// CSV-FLAT - Convert comma-separated data into a flat data file. // Prompts for record length and terminator(s), both of // which are optional. Field widths and initial (default) // separator(s) are determined by scanning the data file. // Implements {EDIT, Convert, CSV to Flat...}. // // Originally by: Thomas C. Burt, Greenview Data, Inc. // Change: 24-February-2005 // Change: 23-Nov-2008 Ch. Ziemski: Help-Button added. // Last change: 08-December Ch. Ziemski: Try to fix problems with user defined delimiters. See "CZ"-markers. // Seems to work now. But has to be tested carefully. // Macro has some more little problems, e.g. statistics are show sometimes, but not correct! ... // // Requires: VEDIT 6.10 or later. // // From VEDIT: Select {EDIT, Convert, CSV to Flat...}. // // Description: At startup, the source data file is examined for quotes and // separators. The first 100 data records are scanned to determine // the field lengths and whether or not they are variable (in which // case, the entire file is scanned to determine the largest width // of each field). // // Initial values are shown to the user in a dialog box and // can be changed therein. // // The user may opt to have each record terminated by DOS // newlines, in which case, he may also choose to not have // records padded out to the full record length. // // The source data file is saved to disk before conversion and // remains unaltered, thereafter. The output filename is user // selectable. The default name is "input_path\file_only.FLT". // // // // From OS: vpw -w -s0 [-y | -q] -x CSV-flat.vdm source_filename [-t reclen] [-a output_name] // // Invocation: -A specifies the name of the output file, including optional drive and path. // When not specified, "filename.flt" will be used. // // -T specifies the fixed output record length. Not needed when the fields // in the source records are fixed length nor when the record length // is specified in the data_description file. // // -Y Save the results and exit; otherwise, display the processed // output file from its beginning. // // -Q Run in "quiet" mode as an icon on the task bar. // // Description: Scans the data input file to determine the separator(s) and // field widths. Scans the first 100 records to determine whether // the file is fixed length; if not, which happens when quoted- // comma-delimited fields have been stripped of leading/trailing // blanks, scans the entire file. // Line terminators are deleted by default; this may be overridden // in the interactive dialog box. If either the "-y" or "-q" options // are specified, exits when finished; otherwise, displays the output // file from its beginning. // // Creates the file "flat_filename.FLT" or the filename specified after // the "-a" parameter. The source file remains unaltered. // May be run under WildFile.VDM or WildFWiz.VDM. // ///////////////////////////////////////////////////////////////////////////////////////////// // // Register Usage: // T-Regs // 0-9 Scratch regs; saved and restored // 0 Output filename into/from DialogInput1() // 1 Record length into/from DI1() // 2 Separator string into/from DI1() // 20-30 Saved and restored // 101 Dialog submacro // 102 Unexpected-Error-Break-Out-Trapping submacro // // // NumRegs // #0-#10 Saved and restored // #8 Number of "overflow" fields (exceed capacity of available NumRegs[]) // #10 ID of T-Reg containing data output pathname (21) // #11 ID of T-Reg containing data output path only (22) // #12 ID of T-Reg containing data source pathname (23) // #15 T-Reg for saving starting current directory (26) // #16 T-Reg for source data output path (27) // #17 T-Reg for data description file output pathname (28) // #18 T-Reg for source data input pathname (29) // #20 T-Reg containing delimiting char/string (20) // #21 First numeric register for storing field widths // #23 Field counter // #24 Maximum # fields to be stored into NumRegs // #25 Total # fields // #26 Flag that fields are Quoted and Comma Delimited (QCD) // #27 Flag to append newline chars // #28 Flag to pad final field // #31-#39 Statistics // #41 Status of FileSelector Window at startup // #42 Save original Zoom value // #43 Message Window (Window for data buffer in command mode) // #44 1=trim leading spaces; 2=trim trailing spaces; 3=both // #64 Flag variable (hex values) // 10 for Quoted-and-Comma-Delimited output fields // #66 Save WILDFILE's Locked-in-Macro ID // #67 Input record type (0,1,2,3,5,6), where 6=fixed length record // #87 is the input record size or 0 // #68 Output record type; #88 = output record size or 0 // #69 Zeroed 1st time in; set to 1 on exit; for running under WildFile // #75 Buffer number for translated output - the "standard" output buffer. // #78 Record counter // #87 Input record size or zero (see #67) // #88 Output record size or zero (see #68) // #90 Source data file's buffer ID // #91 Wildfile-pass counter // #99 Flag - 0x57495C44 ("WILD") when macro is running // #100 Filename position marker // #103 T-Reg this macro is running in // #105 Save/restore Visual_Macro() value // #106 Temp // #129-#256 First 128 field widths (WinVers) // // Preserves NumRegs[0-59] and [70-99] except that #99 is cleared // when this macro is running from the default T-Reg[100||122]. // ////////////////////////////////////////////////////////////////////////////// // // // Execution starts here. // // // ////////////////////////////////////////////////////////////////////////////// #20 = 20 // ID of delimiter register #103 = Macro_Num // #103 = T-Reg # this macro is in if (#103==100||#103==122){#99=0} // Clear WildFile flag when running in // T-Reg[ 100 || 122 ] if (#99==0x57495C44) { // When WILDFILE is running... if (!#91){ // #91 is zeroed (once only) by Wildfile #69 = 0 // 1st time in } #91++ // Update Wildfile pass counter // (May be useful for debugging...) } else { #69 = 0 } // // Check version # >= 6.10, first time in. // Set #24 = max # of indexed fields: // = 100 for 6.10 - 6.12; // = 128 for 6.13 and later. // Set #21 = first numeric register for storing field widths. // // if (#69==0) { #100 = 610 // #100 = minimum version number if (Version_Num<#100) { Reg_Push(0,1) Reg_Set(1,/CSV to "flat" file macro requires VEDIT version /) #1 = #100/100 #2 = Remainder ITOA(#1,1,LEFT+NOCR+APPEND) Reg_Set(1,".",APPEND) ITOA(#2,1,LEFT+NOCR+APPEND) Reg_Set(1," or later.",APPEND) Alert() Dialog_Input_1(0,`"Error","|@(1)"`) Reg_Pop(0,1) if (Is_Quiet) { XALL(1) } else { Break_Out(EXTRA) } } if (Version_Num>=613) { #21 = 129 // First numreg for storing field widths #24 = 128 // Max # field widths to be stored into NumRegs // #24 = 2//128 // Max # field widths to be stored into NumRegs // #21 = 257 - #24 // First numreg for storing field widths } else { #21 = 130 // First numreg for storing field widths #24 = 100 // Max # field widths to be stored into NumRegs // #24 = 2//100 // Max # field widths to be stored into NumRegs } // Caveat Programmer! } // // START - Save NumRegs[0,10]. // Set necessary config values, saving setting in NumRegs. // Save NumRegs[0-59] and [70-99]. // Save T-Regs[0-9] and [20-30]. :START: Num_Push(0,10) #0 = Config(U_AUTO_CFG,0) // Turn off auto-save-config #1 = Config(F_AUTO_SAVE,0) // Turn off auto-save-changes every x minutes #2 = Config(F_OVER_MODE,0) // Allow insert/delete even for fixed-records #3 = Config(F_F_TYPE) #4 = Config(F_REC_HEAD) #5 = Config(E_EXP_TAB) // Tabs complicate COBOL processing #6 = Config(E_RETAB_BK) #7 = Config(E_RETAB_FILL) #8 = Config(S_E_MORE,0) #9 = Config(D_DSP_WRAP,0) #66 = Reg_Lock_Macro // Save WILDFILE's locked-in-macro ID Num_Push(0,59) // Save numeric regs used herein Num_Push(70,99) // Except for #60-#69 Reg_Push(0,9) Reg_Push(20,30) call("SETREGS") // Copy subroutine macros into T-Regs[] /////////////////////////////////////////////////////////////////////////////////// // // Save some state values. // Hide any FileSelector Window. // #41 = 0 // Haven't changed status of FileSelector Window yet if (IsFileSelector) { // When FileSelector Window is visible... #41 = 1 // Remember this fact... Do_Visual("\ME\VI") // And then toggle the window into hidden mode } #42 = Is_Zoomed // TRUE if edit windows are maximized #105 = vm; vm(CLEAR) //Reg_Lock_Macro(102) //////////////////////////////////////////////////////////////////////////////// // // File Handling. // // Set #90 = source data file's buffer ID. Normally, the current edit buffer // is considered to be the source data file. It will be closed and // reopened in read-only mode. // // Set #75 = buffer number for translated output; referred to herein as // the standard output buffer. // #43 = Buf_Num_Window //#43 = message window #100 = 0 #90 = #75 = Buf_Num // #90 = current buffer # #64 = 0 // Flag variable (qcd, only current option) if (#100) {Win_Switch(#43)} Win_Zoom() // Max sized window Win_Clear() if (#100){ ws($) } // // Make the data file's output directory the current default directory. // #15 = 26 // T-Reg[] to hold current default directory #16 = 27 // T-Reg[] to hold source data file's output path #17 = 28 // T-Reg[] to hold description file's output pathname #18 = 29 // T-Reg[] to hold source data input pathname Reg_Set(#15,Cur_Dir) // Save current directory Reg_Set(#16,Path_Only) // Get source data file's output path Reg_Set(#18,Input_File) if (Is_Open_Write) { Ch_Dir(Path_Only) } //////////////////////////////////////////////////////////////// // // // Invoking the Dialog // // // //////////////////////////////////////////////////////////////// Update() // // Setup the default output (.FLT) filename in T-Reg[0]. // Reg_Set(0,pathonly) Reg_Set(0,`\`,APPEND) Reg_Set(0,fileonly,APPEND) Reg_Set(0,`.flt`,APPEND) // // Display dialog except when invoked by "-x" or when // running under WildFile. // if (!Is_Auto_Execution && #99!=0x57495C44 ) { Reg_Set(101,~#106 = DI1(0,^`CSV to Flat`, `This function converts the current file (assumed to be "CSV" data) into a "Flat" file by removing field separators (delimiters) and padding short data items to the width of the corresponding field. The field widths are automatically determined by inspecting the data file. The CR/LF at ends of records can be removed or kept. The converted file will be saved under a new name:`, `??&Output:`, `Optionally, a longer record length can be specified and shorter records will be padded with spaces.`, `??Record &length:`, `Specify the current field separators from the common choices, or enter the separator string.`, `??&Separator (string):`, `.g.l()&User specified separator`, `()&Quote && Comma Delimit`, `()&Tab delimit`, `.g.h.r[]&Keep newlines`, `[]&Pad with spaces to record length`, `[&Run]`, `[&Help]`, `[&Cancel]`^, SET+APP+CENTER,0,0) ~) // `[&Describe]`, // // Set default button to [Describe] or [Run]. // (Not implemented). // if (false) { if (Buf_Switch(Buf_Free(EXTRA),EXTRA|NOMSG)7){ // Don't accept record lengths < 8, here ItoA(#3,1,LEFT|NOCR) } else { if (#3==0) { Reg_Set(1,"Enter, if known; else, longest line will be used.") }} // // Set T-Reg[2] = separator string or name. // Reg_Set(2,@(#20)) // T-Reg[2] sets/gets separator string if (#9==3) { // Tab delimited? Reg_Set(2,"tab") //or(3) char_dump(0x9,NOCR) or(CLEAR) //CZ? T-Reg 3 ??? Why that? Where used later? } // // Set radio button and check boxes initial values. // #26 = (#9==2) // QCD flag #0=#9 //CZ+ (was missing!?) //CZ: if #9 is <1 (invalid) then radio button is not set. User has to decide what to do. See below as well. #1=1 //CZ: Keep (was: Append) Newlines checkbox #2=1 // Pad with spaces checkbox #68 = 6 // Fixed length output records if (! Is_Auto_Execution ){ // When not invoked from the command prompt... while(True) { //CZ+ process new help button call(101) // Invoke the dialog // Numeric register[106] = index of pushed button // {0,1,2,3} = {,[Run],[Help],[Cancel]} // Buf_Num = #90 = source data buffer // // [Help]. // if (#106==2){ Help("Converting") // That help page needs another click on a link, but it's the best entry I found. } else { break } } // // [Cancel] or . // if (#106==0 || #106==3 ){ Buf_Switch(#90,NOMSG) Goto DONE } //////////////////////////////////////////////////////////////// // // // Processing the Dialog Return Values // // // //////////////////////////////////////////////////////////////// if (#0<=1) { //CZ+ if still no choice of delimiter done (via radio button) if (Reg_Compare(2,@(#20))<>0){ //CZ+ but a string has been given #0=1 //CZ+ set as "user defined delimiter" } //CZ+ } //CZ+ if ((#0==1) || (#0==2)) { //CZ+ user defined delimiter Reg_Set(#20,@2) //CZ+ T-Reg[2] has separator string call("GETPARMS") //CZ+ now try again to get the parameters of data } //CZ+ if (#0==2) { //CZ+ quoted and (?comma) delimited #26=1 //CZ+ QCD flag } //CZ+ if (FALSE) { //CZ+ set to TRUE for debugging M("## T-Reg 0-2:") tn() rt(0) tn() rt(1) tn() rt(2) tn() M("## #0=") .#0 M("## #20=") .#20 M("## T-Reg #20>>>") rt(#20) m("<<<") tn() GK("press any key...") } #27 = #1 //CZ: Flag to keep (was: append) newline chars #28 = #2 // Flag to pad final field if (!#27){#28=1} // Must pad fixed-length records when no newlines #88 = AtoI(1,NOERR) // Output record length } // fi (!Is_Auto_Execution) (Dialog processing) if (#27){ // When appending newlines... if ((#0 = Config(F_F_TYPE))>5){ // #0 = input type = record mode? #68 = 0 // Yes, select DOS newline characters } else { // Else use input type for the output type #68 = #0 // #68 = #0 = input type, from "if", above } } ////////////////////////////////////////////////////// // // // Sign On // // // ////////////////////////////////////////////////////// #10 = 21 // Use T-Reg[21] for data output pathname Buf_Switch(#90,NOMSG) // Ensure the source file is the current edit buff if (#69==0) { if (!Is_Quiet) { // If first time (WildFile) && interactive mode if ( OS_Type == 1 && ( Screen_Lines < 15 || Screen_Cols < 57 )) { Screen_Size(15,57) Screen_Init() } else { Win_Clear() } Type_Newline() #0 = ( Screen_Cols - 57 ) >> 1 Type_Space(#0) Message('*********************************************************\n');TS(#0) Message('* CSV-FLAT.VDM 02/24/2005 *\n');TS(#0) Message('* Make "flat" data file from CSV file. *\n');TS(#0) Message('*********************************************************\n');TN() // // If no file open, prompt for filename and open it. // Reg_Set(3,"file to convert") if (!File_Size) { repeat(ALL) { Reg_Set(3,"Enter name of ",INSERT) Reg_Set(3,": ",APPEND) Type_Newline() Reg_Type(3) Get_Input(#10,"",NOCR) if (File_Exist(@(#10))) { Break } Alert() Message("\nFile not found; please try again or to cancel.\n") } File_Open("|@(#10)",NOEVENT) #90 = Buf_Num } }} //if (wstat($)<0){wr($,5,top)};Reg_Lock_Macro(CLEAR);ws($);#100=1;update() ? if ( File_Size == 0 ) { Goto DONE } // // For Auto-Execution, set Current Directory to source data file's input directory. // Buf_Switch(#90,NOMSG) Reg_Push(0,1) Reg_Set(0,INPUT_FILE) Buf_Switch(Buf_Free(EXTRA),NOMSG) Reg_Ins(0) EOF() Replace("|{\,:}","",REVERSE) if(Match_Item==2){ Ins_Text(":\") } Del_Line() Reg_Copy(1,0) if ( Is_Auto_Execution ) { Ch_Dir(@(1)) } Reg_Pop(0,1) Buf_Quit(OK|NOMSG) Buf_Switch(#90,NOMSG) // // Set T-Reg[#10] = data output pathname: // path\fname.flt (normally) or // user selected name from DI1() or // "-a" parameter for Auto-Execution ("-x"). // if (wstat($)<0){wr($,5,top)};Reg_Lock_Macro(CLEAR);ws($);#100=1;update() ? if ( Is_Auto_Execution ) { // // Auto_Execution && "-a" // if ( Is_SaveAs ) { Reg_Set(#10,PATH_NAME) // T-Reg[#10] = user-specified output pathname // // Auto_Execution Default: Input_Path\File_Name.CSV // } else { Reg_Set(#10,PATH_ONLY) // = path Reg_Set(#10,"\",APPEND) // = path\ Reg_Set(#10,FILE_ONLY,APPEND) // = path\fname Reg_Set(#10,".FLT",APPEND) // = path\fname.FLT } } else { // // Interactive name from Dialog_Input_1() // Reg_Set(#10,@0) } // // Set T-Reg[#11] = output_path\. // #0 = Buf_Num #11 = 22 Buf_Switch(Buf_Free(EXTRA),NOMSG) Reg_Ins(#10) //buf[] = full_path_name (output) Search("\",BEGIN+ALL+ADVANCE+NOERR) //Skip over pathname Reg_Copy(#11,0) //T-Reg[#11] = output_path\ Buf_Quit(OK) Buf_Switch(#0,NOMSG) // // Check for missing/empty data file. // if ( FSize == 0 ) { Reg_Empty(1) Out_Reg(1,APPEND) Message("\n***** Missing or empty data file ") Reg_Type(0) Message("\n") Out_Reg(CLEAR) // Call("ERRMSG") if (Is_Quiet) { XALL(1) } else { if (#99==0x57495C44) { break_out } else { break_out(EXTRA) }} } //////////////////////////////////////////////////////// // // // Determine tentative input EOL type or record size. // // Set #67 = input EOL-type: {0,1,2,3,4,6} and // // #87 = input record size or 0. // // // //////////////////////////////////////////////////////// // Note: 0-4 are config/menu values for Vedit's terminated "file-type". // 6 = fixed-length record. // #67 = Config(F_F_TYPE) #87 = 0 if ( #67 > 6 ) { #87 = #67 #67 = 6 // Fixed length records } // // Set T-Reg[#12] = source_data_pathname. // #12 = 23 Out_Reg(#12) Name_Read(EXTRA+NOMSG+NOCR) Out_Reg(CLEAR) ////////////////////////////////////////////////// // // // Open the output buffer. // // Set #75 = buffer ID. // // Configure preliminary record-type/size. // // // ////////////////////////////////////////////////// Buf_Switch(Buf_Free) File_Open_Write(@(#10),OVERWRITE+OK+NOMSG) #75 = Buf_Num Config(F_OVER_MODE,0,ALL) if ( #67 != 6 ) { Config(F_F_TYPE,#67,LOCAL) } else { if ( #87 > 7 ) { Config(F_F_TYPE,#87,LOCAL) } else { Config(F_F_TYPE,7,LOCAL) Config(F_CUST_REC_SIZE,#87,LOCAL) }} ////////////////////////////////////////////////// // // // Close the source data file. // // Reopen in Read-only mode. // // Set #90 = buffer ID. // // // ////////////////////////////////////////////////// //if (wstat($)<0){wr($,5,top)};Reg_Lock_Macro(CLEAR);ws($);#100=1;update() ? Buf_Switch(#90) if (Is_Altered){ File_Close(NOMSG) } else { Buf_Empty(OK) } //if (wstat($)<0){wr($,5,top)};Reg_Lock_Macro(CLEAR);ws($);#100=1;update() ? File_Open("|@(#12)",BROWSE+OVERWRITE+OK+NOMSG) #90 = Buf_Num if ( #67 != 6 ) { Config(F_F_TYPE,#67,LOCAL) } else { if ( #87 > 7 ) { Config(F_F_TYPE,#87,LOCAL) } else { Config(F_F_TYPE,7,LOCAL) Config(F_CUST_REC_SIZE,#87,LOCAL) }} ////////////////////////////////////////////////////////////// // // // Preprocessing // // First pass: determine maximum field widths. // // // ////////////////////////////////////////////////////////////// // // // If the field widths of each of 100 records remain // // the same, assume the source is basically "flat"; else // // scan the entire file determining the maximum width of // // each field. // // // // If the number of fields per record are less than the // // number of numeric registers available, use the numeric // // registers; else use an extra edit buffer (ugh!). // // // ////////////////////////////////////////////////////////////// #0 = 130 #4 = 0 // Current field beginning column // // Determine max # of indexible fields. // #1 = min(#24,#25) // // Reset all reserved NumRegs. // #23 = 0 for ( #0 = #21; #0 < #21 + #1; #0++ ) { #@0=0 #23++ } // // Also, initialize any overflow fields to zero in a work buffer. // Exit: N-reg[8] = # overflow fields. // #8 = 0 // N-Reg[8] if (#24<#25){ BS(XBUF1,EXTRA) DB(0,FSize) #8 = #25 - #24 for (#0=0;#0<#8;#0++){ IT("0,") } BoF() BS(#90) } // // Set T-Reg[0] to quote-mark when "quoted and comma delimited". // if (#26) { // Quoted and comma delimited? Buf_Switch(XBUF1) // Switch to work buffer Del_Block(0,File_Size) // Be safe Reg_Ins(#20,BEGIN) // "," normally Reg_Copy_Block(0,Cur_Pos,Cur_Pos+1) // T-Reg[0] contains one of ["'`] Buf_Quit(OK) // Release the work buffer } Buf_Switch(#90) // Enter source data buffer Begin_Of_File() // Start at the beginning #6 = 0 // Overall flag for changes in field widths #78 = 0 // Line counter //if (wstat($)<0){wr($,5,top)};Reg_Lock_Macro(CLEAR);ws($);#100=1;update() ? // // Perform following loop 100 times if widths are constant or // for the entire file, otherwise. // while(!At_EoF) { #78++ // #78 = Current Line # // // Determine EoL_Pos even when in record mode. // Save_Pos() Line(1,NOERR) if ( Config(F_F_TYPE) < 6 ) { if (! At_EoF ) { char(-NewLine_Chars) } else { if ( Cur_Char == 0x1a ){ char(-NewLine_Chars) }} } #3 = Cur_Pos Restore_Pos() // // Advance past initial quote-mark when "quoted and comma delimited". // if (#26) { Match(@0,ADVANCE) } // #7=0 // Flag for field width changes in current line #23=0 // Field counter // // Process indexible fields. // for (#0=#21;#0<#21+#1;#0++){ #4=CP // Save current position // // Advance to next delimiter in current line; exit loop when no more. // Search_Block(@(#20),#4,#3,ERRBREAK) #9 = Chars_Matched // #9 = size of delimiter string // // Update numeric register and set "change" flag, perhaps. // #5 = CP-#4 // Determine field width if ( #@0 < #5 ){ #@0 = #5 #7 = 1 } Char(#9) // Advance past delimiter(s) #23++ } // rof each indexible field... // // Process any buffered overflow fields. // if (#23<#25){ // If # indexible fields < total # fields... // // For each remaining overflow field... // for (#0=0;#0<#8;#0++){ #4 = CP // // Advance to next delimiter in current line; exit loop when no more. // Search_Block(@(#20),#4,#3,ERRBREAK) #9=Chars_Matched // #9 = size of delimiter string #5=CP-#4 // Determine field width // // Update overflow field and set "change" flag, perhaps. // Buf_Switch(XBUF1) // Switch to work buffer #10=NE() if (#10<#5){ DC(CMAT) NI(#5,LEFT|NOCR) #7=1 } else { C(CMAT) } Match("|[|w],",ADVANCE) Buf_Switch(#90) // Switch back to data buffer Char(#9) // Advance past delimiter(s) #23++ } // rof each remaining overflow field BS(#90) } // fi any overflow fields BS(#90) #5=#3-CP // #5 = width of final field, usually if (#26){#5--} // Correction needed when QCD // // Update final field width and set "change" flag, perhaps. // if (#8==0) { // When just indexible fields... #0 = #21 + #25 - 1 // #0 = index of final field if ( #@0 < #5 ){ #@0 = #5 #7=1 // Set change flag, this record } } else { // When overflow field(s)... Buf_Switch(XBUF1) // Switch to work buffer #10=NE() if (#10<#5){ DC(CMAT) NI(#5,LEFT|NOCR) #7=1 } BoF() // Reset edit position to beginning of work buffer Buf_Switch(#90) // Switch back to data buffer } // // Finish up preprocessing current record. // #6|=#7 // Update change flag, any field, any record, so far Line(1,NOERR) // Advance to next data record if ((#78>=100 && #6==0)||At_Eof){break} // Assume that no changes in any fields for 100 records // implies all fields are the same for the entire file } // elihw !At_EoF // // Compute #2 = sum of all field widths = "actual" record length. // #1 = min(#24,#25) #2 = 0 for ( #0 = #21; #0 < #21 + #1; #0++ ){ #2 += #@0 // #2 = sum of all indexed field widths } if (#8){ // Now include any "overflow" field widths Buf_Switch(XBUF1) BoF() for (#0=0;#0<#8;#0++){ #2 += Num_Eval(ADVANCE) match("|[|w],",ADVANCE) } } // // Adjust final field depending on Newlines, Padding, specified record // length and actual record length. // Enter: #27 = flag to append newline chars. // #28 = flag to pad final field. // #88 = specified record length (may be 0). // #2 = actual record length. // if (#8==0){ #0 = #21 + #1 - 1 // #0 = index of final field if (#27&&!#28) { // Appending newlines but not padding final field? #@0 = 0 // Then reset final field width } if (#28 && #88>7 ) { // Padding final field to specified record length (#88)? if (#88 > #2) { // Adjust when specified record length > actual length #@0 += (#88-#2) // Parentheses required (Sigh!) } } } else { if (s(",",REVERSE|COUNT|NOERR,2)==2){ Char(1) } else { BoF() } #9 = Num_Eval() if (#27&&!#28) { // Appending newlines but not padding final field? Del_Char(Chars_Matched) Num_Ins(0,LEFT|NOCR) // Then reset final field width so no padding will be added } if (#28 && (#88>7)) { // Padding final field to specified record length (#88)? if (#88 > #2) { // Adjust when specified record length > actual length Del_Char(Chars_Matched) Num_Ins(#9+#88-#2) } } } if (#68>5&>#88){#88=#2} // Reset "specified" record length to actual length // // Reconfigure output filetype. // Buf_Switch(#75) if (#27){ // When newline terminators... #0 = #68 // Specify what kind: [DOS,Unix,Mac,...] } else { #0 = #88 // Otherwise, set to fixed record length } Config(F_F_TYPE,#0,LOCAL) /////////////////////////////////////////////////////////////// // // // Statistics Initialization // // // // Set #76 = (approx) record size. // // #77 = (approx) # records in the file. // // #31 = # records per reporting cycle {100 or 10 or 1} // // // /////////////////////////////////////////////////////////////// Buf_Switch(#90,NOMSG) //Switch to source data file BoF() if (#67>5){ //If fixed-length records... #76 = #87 //Default record size } else { //Else file should have newlines... #0 = Cur_Pos Line(100,NOERR) //Advance a few lines #76 = (Cur_Pos - #0)/100 //Approx line size Goto_Pos(#0) //Restore pos } #77 = (File_Size-#0) / #76 //#77 = # of records in file #31 = 100 if (#77 < 500){ #31 = 10 } if (#77 < 50){ #31 = 1 } // // Show life by displaying "Processing first xxx data records..." message. // // if (!Is_Quiet) { Win_Hor(1) Message("Input file: ") // Display input drive:\path\name Reg_Type(#18,0) Win_EOL() Type_Newline() Buf_Switch(#75,NOMSG) Name_Write(EXTRA) // Ditto for output file Buf_Switch(#90,NOMSG) if (false){ if (Reg_Size(#65+18)){ ***** error ***** Message("Error file: ") // Ditto for error error file Reg_Type(#65+18,0) ***** error ***** Win_EOL() Type_Newline() } } // if (FALSE) Message("Processing first ") Num_Type(#31,LEFT+NOCR) Message(" data records... ") } // // More statistics. // #36 = File_Size #39 = Reg_Free // Do nothing for record #0 // // Set T-Reg[9] to quote-mark when "quoted and comma delimited". // if (#26) { // Quoted and comma delimited? Buf_Switch(XBUF1) // Switch to work buffer Del_Block(0,File_Size) // Be safe Reg_Ins(#20) // "," normally Reg_Copy_Block(9,Cur_Pos,Cur_Pos+1) // T-Reg[9] contains one of ["'`] Buf_Quit(OK) // Release the work buffer } ////////////////////////////////////////////////////////////////////////////// // // Main Processing loop. // #77 = #75 // Initially, output buffer = standard output buffer #78 = 0 // Record counter #32 = Time_Tick BS(#90,NOMSG) // Enter source data buffer Begin_Of_File() // Start at the beginning while(!AtEOF){ // For each record // // Show progress message every 100 or 10 or 1 iterations. // if (!IsQuiet&&!(#78%#31)){ call(#39) // T-Reg(#39) is empty, initially #39=62 // Next time it'll have the right stuff } #78++ // Increment record # // // Initialization for each record: // copy current record into the output buffer; // delete initial quote-mark when QCD; // rewind any "overflow" buffer. // // // First, copy current record into T-Reg[0]. // if (#67<6){ RCB(0,cp,EoLPos,NoRestore) L(1,NOERR) } else { RC(0,1,NoRestore) } // // Rewind any "overflow" buffer. // if (#8){ BS(XBUF1) BoF() } // // Copy current record from T-Reg[0] into output buffer and // set N-Reg[3] to beginning of the current record; // set Marker[4] past the end of the record; // delete any initial quote mark. // BS(#77,NOMSG) #3 = CP RI(0) Set_Marker(4,CP) // Adjustable marker at end of data GP(#3) // Goto start of data if (#26) { DC(1) } ////////////////////////////////////////////////////////////////////////// // // Field processing loop(s). // // // // Delete delimiters; pad short fields. // #1 = min(#24,#25) // #1 = # indexed fields for ( #23=0; #23<#1; #23++ ) { #0=#21+#23 // #0 = field index #0=#@0 // = field width // // Advance to next delimiter in current line; exit loop if no more. // #3=CP // #3 = field's starting position SB(@(#20),#3,Marker(4),ERRBREAK) #9 = CMat // #9 = size of matched delimiter string #5 = CP - #3 // #5 = actual field width #6 = #0 - #5 // #6 = # padding spaces needed if (#6>0){ IC(' ',COUNT,#6) } DC(#9) // Delete delimiter(s) } // rof each indexible field... // // Now process "overflow" fields. // if (#8) { // // Set #0 = output field width. // Buf_Switch(XBUF1) #0 = Num_Eval(ADVANCE|SUPPRESS) Match(",|[|w]",ADVANCE) // // Advance to next delimiter in current output line; exit loop if no more. // BS(#77) // Switch to the output buffer #3=CP // #3 = field's starting position SB(@(#20),#3,Marker(4),ERRBREAK) // Find next delimiter (string) #9 = CMat // #9 = size of matched delimiter string #5 = CP - #3 // #5 = actual field width #6 = #5 - #0 // #6 = # padding spaces needed if (#6>0){ IC(' ',COUNT,#6) } DC(#9) // Delete delimiter(s) } // fi overflow fields... // // Process final field. // // Note: delimiter after final field either differs from // from preceding delimiters (QCD) or is absent entirely (CSV). // #3 = CP GM(4) // GotoEoL() if (#26){ // Correction needed when QCD DC(-1) } #5 = Cur_Pos - #3 // #5 = actual width of final field // if (#8==0){ // When final field is indexed... #0=#21+#23 // #0 = index of final field #0=#@0 // #0 = desired width of final field } else { // When final field is a "overflow" field Buf_Switch(XBUF1) // Switch to work buffer #0=NE() BS(#77) // Switch back to data output buffer } #6 = #0 - #5 // #6 = # padding spaces needed if (#6>0){ IC(' ',COUNT,#6) } // // Insert newline char(s), if so specified. // if (#27){IN} // // Switch back to the source data buffer. // BS(#90,NOMSG) } // Main processing loop ///////////////////////////////////////////////////////////////////////////// // // Set true to view final statistics. // if ( FALSE ) { if (!Is_Quiet) { //CZ: why not as option in dialog? Win_Hor(1) Message("Converted records: ") Num_Type(#78,NOCR) Message(" (") Num_Type(100,LEFT+NOCR) Message("%)") Win_EOL() Message("\nActual elapsed time: ") if ( 1 ) { Num_Type((Time_Tick - #32 + 500)/1000,LEFT+NOCR) Message(" seconds.") // Pause at end of message } else { Num_Type(Time_Tick - #32,LEFT+NOCR) Message(" milliseconds.") // Pause at end of message } if (!Is_Option(y)){ Get_Key("") } }} ////////////////////////////////////////////////////////////////////////////// // // Clean up // Buf_Switch(#75,NOMSG) // Switch to the main output file if (#99!=0x57495C44) { // Unless WildFile... File_Save(BEGIN+NOMSG) // Save output file; goto BoF (quickly) } else { // When WildFile... Buf_Close(DELETE|NOMSG) } // // Release source data file. // Reopen normally unless terminating or // running under WildFile. // Buf_Switch(#90,NOMSG) #0 = Config(F_F_TYPE) if (#99!=0x57495C44){ File_Quit(OK) } else { Buf_Quit(OK|DELETE) } if (#99!=0x57495C44 && !Is_Auto_Execution && !(Is_Option(y) || Is_Option(q))){ File_Open("|@(#12)",OK+NOMSG) Config(F_F_TYPE,#0,LOCAL) } // // Restore text and numerical registers and then terminate. // //{ :DONE: // if (wstat($)<0){wr($,5,top)};Reg_Lock_Macro(CLEAR);ws($);#100=1;update() ? Reg_Lock_Macro(CLEAR) //Disable further error trapping if (Win_Status('h')>=0){ Win_Delete('h') //Remove any Help window } #0=Win_Num Win_Switch(#43,NOMSG) // Switch to the data window, if not there now if (!#42){Win_Zoom(CLEAR)} // Restore original window zoom state Win_Switch(#0,NOMSG) if (#41) { //If the FileSelector Window was hidden Do_Visual("\ME\VI") //Reshow it now } Ch_Dir(@(#15)) //Restore original "current" directory if (#99==0x57495C44){ Buf_Switch(MAINBUF|NOMSG) } else { Buf_Switch(#75,NOMSG) //Switch to main output buffer } Reg_Empty(101) //Empty dialog submacro Reg_Empty(102) //Empty unexpected breakout trapper Reg_Pop(20,30) Reg_Pop(0,9) Num_Pop(70,99) //Restore numeric regs Num_Pop(0,59) Config(U_AUTO_CFG,#0) //Restore user's config values Config(F_AUTO_SAVE,#1) Config(F_OVER_MODE,#2) // no Config(F_F_TYPE,#3) // no Config(F_REC_HEAD,#4) Config(E_EXP_TAB,#5) Config(E_RETAB_BK,#6) Config(E_RETAB_FILL,#7 & 0xff ) Config(S_E_MORE,#8) Config(D_DSP_WRAP,#9) Num_Pop(0,10) //Restore remaining user numregs Reg_Lock_Macro(#66) #69 = 1 //WILDFILE flag for next time in if (Is_Option(y)) { XAll(NOMSG) } //If "-Y" invocation, exit now if (#99==0x57495C44) { //If WILDFILE macro running if (!Is_Quiet) { Type_Newline() } } if (#105) { vm(NOMSG) } return // // ShowHelpWindow - // :ShowHelpWindow: #0=Win_Num Win_Switch(#43) // Switch to the data window, if not there now if (!#42){Win_Zoom(CLEAR)} // Restore original window zoom state if (Win_Border>=2) { Win_Reserved('h',1,BOTTOM+NOBORDER) } else { Win_Reserved('h',1,BOTTOM) } Win_Switch('h') Win_Color(112) Win_Clear() Message("[VISUAL EXIT] (<\>) when done [VISUAL ESCAPE] (<\>) to quit",EXTRA) Win_Switch(#43) //Switch back to message window return // // Toggle FileSelector Window // :ToggleFileSel: Do_Visual("\ME\VI") return // // BADLAY - NYI // :BADLAY: //} DONE() /////////////////////////////////////////////////////////////////////////////// // // SETREGS - Copy submacros into T-Regs[]. // //{ :SETREGS: // // T-Reg[62] - Progress Display. // // First time, display estimated processing time. // Also display persistent "Converted records: " // Every time, display count and percentage of records processed. // Reg_Set(62,` if (#78==#31){call(63)} WH(#33) NT(#78,NOCR) // count #37=FSize-CP if (#37>1000000){#34=100-(#37/(#36/100))} else{if (#36==0){#34=100} else{#34=100-((100*#37)/#36)} } Message(" (") NT(#34,LEFT+NOCR) // percentage Message("%)") `) // [62] Progress Display // // T-Reg[63] - Progress Display (Initial Message). // // Display estimated processing time. // Also display persistent "Converted records: " // Empties itself on exit. // Reg_Set(63,` #35=TT WH(1) Message("Estimated processing time is ") #34=((#35-#32)*((#36/(#36-FSize+CP)))+500)/1000 NT(#34,LEFT+NOCR) Message(" seconds\n") Message("Converted records: ") #33=WH RE(MN,EXTRA) `) // [63] Progress Display (Initial Message) // // T-Reg[102] - Unexpected error breakout trap. // Restore Vedit's original state, then // terminate. // Reg_Set(102,` Reg_Lock_Macro(CLEAR) //Disable further error trapping // // Release and reload source data file. // Buf_Switch(#90,NOMSG) File_Quit(OK) File_Open(@(#18),OK+NOMSG) // // Save output file; display it from BoF. // Buf_Switch(#75,NOMSG) //Switch to the main output file File_Save(BEGIN+NOMSG) //Save output file; goto BoF (quickly) // // Remove any Help window. // if (Win_Status('h')>=0){ Win_Delete('h') //Remove any Help window } // // Reshow the FileSelector Window. // if (#41) { //If the FileSelector Window was hidden Do_Visual("\ME\VI") //Reshow it now } // Ch_Dir(@(#15)) //Restore original "current" directory Reg_Empty(101) //Empty dialog submacro Reg_Pop(20,30) //Restore T-Regs Reg_Pop(0,9) Num_Pop(70,99) //Restore numeric regs Num_Pop(0,59) Config(U_AUTO_CFG,#0) //Restore user's config values Config(F_AUTO_SAVE,#1) Config(F_OVER_MODE,#2) // no Config(F_F_TYPE,#3) // no Config(F_REC_HEAD,#4) Config(E_EXP_TAB,#5) Config(E_RETAB_BK,#6) Config(E_RETAB_FILL,#7&0xff) Config(S_E_MORE,#8) Config(D_DSP_WRAP,#9) if (#99==0x57495C44) { //If WILDFILE macro running Config(F_F_TYPE,#4) Config(F_REC_HEAD,#5) } Num_Pop(0,10) //Restore remaining user numregs Reg_Lock_Macro(#66) //Restore caller's lock, perhaps #69 = 1 //WILDFILE flag for next time in if (Is_Quiet || Is_Option(y)){ XALL(1) } else { if (#105){ vm(SET) } Reg_Empty(Macro_Num,EXTRA) } `) // [102] Unexpected Error Breakout //} End SetRegs() return //CZ+ was missing!? /////////////////////////////////////////////////////////////////////////////// // // GETPARMS - Process data file to determine initial parameters. // Return: 0 or negative, if invalid // 3 if tab delimited // 2 if quoted and comma delimited (Includes any of // '"', "'", "`" surrounding any separator) // 1 otherwise // // #3 = output record length guestimate. // #25 = # fields/record. // bufnum = #90 = source data buffer // // T-Reg[#20] = delimiter string. // //{ :GETPARMS: //gk("getparms") //CZ+ for debugging Buf_Switch(#90,NOMSG) // Enter source data buffer BoF() #0 = 0 // invalid/type delimiter flag #3 = EoL_Pos // Record size #25 = 1 // # fields/record ////////////////////////////////////////////////////////////// // // // Some data file validation. // // // ////////////////////////////////////////////////////////////// if (File_Size==0){ return(0) } //CZ: Attention: Originally only these hard coded delimiters were allowed. //CZ: A user defined delimiter from dialog didn't work correctly!!! //CZ: Part of a fix is below, the other part is to call this after DI1() again. Reg_Set(1, @(#20)) //CZ+ needed because double redirection doesn't seem to be evaluated below //if (!sb(/|{",',`,\,^,%,*,||,:,;,|h09,|,}/,1,10,LINE_SET|NOERR)) //CZ~ if (!sb(/|{",',`,\,^,%,*,||,:,;,|h09,|,,|@(1)}/,1,10,LINE_SET|NOERR)) { //CZ~ return(-1) } if (Cur_Line!=1){ return(-2) } ////////////////////////////////////////////////////////////// // // // Delimiter Determination. // // // ////////////////////////////////////////////////////////////// #1 = Match_Item #2 = (Cur_Char == 0x9) rcb(1,Cur_Pos,Cur_Pos+1) // // Return 2 if Quoted and Comma Delimited. // if ( #1 <= 3 ) { // Quote marks {",',`} ? if ( Cur_Col != 1 ) { // Need to start in column 1 return(0) // } if (sb("|@(1)|s|@(1)",Cur_Pos+1,EoL_Pos,NOERR) == 0){ return(0) } else { rcb(1,Cur_Pos,Cur_Pos+Chars_Matched) #0 = 2 } } // // Return 3 if tab delimited. // if (#2){ #0 = 3 } // // Return 1 for all other cases. // if (#0==0){ #0 = 1 } ////////////////////////////////////////////////////////////// // // // Number of Fields/record. // // // ////////////////////////////////////////////////////////////// #1 = Search_Block(@(1),1,1,LINE_SET|BEGIN|ALL) #2 = Search_Block(@(1),2,2,LINE_SET|BEGIN|ALL) if (#1!=#2){ return(0) } else { #25 = #1 + 1 } ////////////////////////////////////////////////////////////// // // // Record Length. // // // ////////////////////////////////////////////////////////////// Goto_Line(2) Char(-Newline_Chars) // At EoL of Line 1 #4 = Cur_Pos() // #4 = length of line 1 Goto_Line(3) Char(-Newline_Chars) // At EoL of Line 2 #5 = Cur_Pos - #4 - Newline_Chars // #5 = length of line 2 if (#5==#4){ #3 = #5 - (#25-1)*Reg_Size(1) if (Reg_Size(1)>1){ // Adjustment when Quoting & Comma Delimiting #3 -= 2 } } else { #3 = 0 } // // Finish up. // Reg_Set(#20,@(1)) // Save delimiter string Return( #0 ) //} End GetParms()