20180418a

python-per-line.py

@@ -3,7 +3,7 @@
 __description__ = "Program to evaluate a Python expression for each line in the provided text file(s)"
 __author__ = 'Didier Stevens'
 __version__ = '0.0.4'
-__date__ = '2018/04/01'
+__date__ = '2018/04/17'
 
 """
 
@@ -21,6 +21,7 @@
   2017/07/23: updated man
   2018/02/05: 0.0.3 added option -i
   2018/04/01: 0.0.4 added support for # to option -o
+  2018/04/17: refactored support for # to option -o
 
 Todo:
 """
@@ -104,7 +105,16 @@ def PrintManual():
 #c#example.txt will write output both to the console (stdout) and file example.txt.
 #g# will write output to a file with a filename generated by the tool like this: toolname-date-time.log.
 #g#KEYWORD will write output to a file with a filename generated by the tool like this: toolname-KEYWORD-date-time.log.
-#c# and #g# can be combined: #cg#.
+Use #p#filename to display execution progress.
+To process several files while creating seperate output files for each input file, use -o #s#%f%.result *.
+This will create output files with the name of the inputfile and extension .result.
+There are several variables available when creating separate output files:
+ %f% is the full filename (with directory if present)
+ %b% is the base name: the filename without directory
+ %d% is the directory
+ %r% is the root: the filename without extension
+ %e% is the extension
+Most options can be combined, like #ps# for example.
 #l# is used for literal filenames: if the output filename has to start with # (#example.txt for example), use filename #l##example.txt for example.
 
 An extra Python script (for example with custom definitions) can be loaded using option -s.
@@ -116,6 +126,8 @@ def PrintManual():
     for line in manual.split('\n'):
         print(textwrap.fill(line))
 
+DEFAULT_SEPARATOR = ';'
+
 def File2Strings(filename):
     try:
         if filename == '':
@@ -196,23 +208,65 @@ def SBC(data, separator, columns, column, failvalue):
     else:
         return data[indices[column - 1] + 1:indices[column]]
 
+class cVariables():
+    def __init__(self, variablesstring='', separator=DEFAULT_SEPARATOR):
+        self.dVariables = {}
+        if variablesstring == '':
+            return
+        for variable in variablesstring.split(separator):
+            name, value = VariableNameValue(variable)
+            self.dVariables[name] = value
+
+    def SetVariable(self, name, value):
+        self.dVariables[name] = value
+
+    def Instantiate(self, astring):
+        for key, value in self.dVariables.items():
+            astring = astring.replace('%' + key + '%', value)
+        return astring
+
 class cOutput():
-    def __init__(self, filename=None, bothoutputs=False, generatefilename=False):
-        self.filename = filename
-        self.bothoutputs = bothoutputs
-        self.generatefilename = generatefilename
-        if self.generatefilename:
-            if self.filename != None and self.filename != '':
-                extra = self.filename + '-'
-            else:
-                extra = ''
-            self.outputfilename = '%s-%s%s.log' % (os.path.splitext(os.path.basename(sys.argv[0]))[0], extra, self.FormatTime())
-        else:
-            self.outputfilename = self.filename
-        if self.outputfilename != None and self.outputfilename != '':
-            self.f = open(self.outputfilename, 'w')
-        else:
-            self.f = None
+    def __init__(self, filenameOption=None):
+        self.filenameOption = filenameOption
+        self.separateFiles = False
+        self.progress = False
+        self.console = False
+        self.f = None
+        if self.filenameOption:
+            if self.ParseHash(self.filenameOption):
+                if not self.separateFiles:
+                    self.f = open(self.filename, 'w')
+            elif self.filenameOption != '':
+                self.f = open(self.filenameOption, 'w')
+
+    def ParseHash(self, option):
+        if option.startswith('#'):
+            position = self.filenameOption.find('#', 1)
+            if position > 1:
+                switches = self.filenameOption[1:position]
+                self.filename = self.filenameOption[position + 1:]
+                for switch in switches:
+                    if switch == 's':
+                        self.separateFiles = True
+                    elif switch == 'p':
+                        self.progress = True
+                    elif switch == 'c':
+                        self.console = True
+                    elif switch == 'l':
+                        pass
+                    elif switch == 'g':
+                        if self.filename != '':
+                            extra = self.filename + '-'
+                        else:
+                            extra = ''
+                        self.filename = '%s-%s%s.log' % (os.path.splitext(os.path.basename(sys.argv[0]))[0], extra, self.FormatTime())
+                    else:
+                        return False
+                if len(self.filename) == 0:
+                    self.separateFiles = False
+                    return False
+                return True
+        return False
 
     @staticmethod
     def FormatTime(epoch=None):
@@ -221,20 +275,38 @@ def FormatTime(epoch=None):
         return '%04d%02d%02d-%02d%02d%02d' % time.localtime(epoch)[0:6]
 
     def Line(self, line):
-        if not self.f or self.bothoutputs:
-            print(line)
-        if self.f != None:
+        if not self.f or self.console:
             try:
-                self.f.write(line + '\n')
-                self.f.flush()
-            except:
-                pass
+                print(line)
+            except UnicodeEncodeError:
+                encoding = sys.stdout.encoding
+                print(line.encode(encoding, errors='backslashreplace').decode(encoding))
+#            sys.stdout.flush()
+        if self.f:
+            self.f.write(line + '\n')
 
     def LineTimestamped(self, line):
         self.Line('%s: %s' % (self.FormatTime(), line))
 
+    def Filename(self, filename):
+        self.separateFilename = filename
+        if self.progress:
+            print(self.separateFilename)
+        if self.separateFiles:
+            oFilenameVariables = cVariables()
+            oFilenameVariables.SetVariable('f', self.separateFilename)
+            basename = os.path.basename(self.separateFilename)
+            oFilenameVariables.SetVariable('b', basename)
+            oFilenameVariables.SetVariable('d', os.path.dirname(self.separateFilename))
+            root, extension = os.path.splitext(basename)
+            oFilenameVariables.SetVariable('r', root)
+            oFilenameVariables.SetVariable('e', extension)
+
+            self.Close()
+            self.f = open(oFilenameVariables.Instantiate(self.filename), 'w')
+
     def Close(self):
-        if self.f != None:
+        if self.f:
             self.f.close()
             self.f = None
 
@@ -267,13 +339,22 @@ def __init__(self, option):
         return
 
 class cOutputResult():
-    def __init__(self, option):
-        self.oOutputOptions = cOutputOptions(option)
-        self.oOutput = cOutput(self.oOutputOptions.filename, self.oOutputOptions.bothoutputs, self.oOutputOptions.generate)
+    def __init__(self, options):
+        if options.output:
+            self.oOutput = cOutput(options.output)
+        else:
+            self.oOutput = cOutput()
+        self.options = options
 
     def Line(self, line):
         self.oOutput.Line(line)
 
+    def LineTimestamped(self, line):
+        self.oOutput.LineTimestamped(line)
+
+    def Filename(self, filename):
+        self.oOutput.Filename(filename)
+
     def Close(self):
         self.oOutput.Close()
 
@@ -335,8 +416,9 @@ def PythonPerLine(expression, filenames, options):
     if options.script != '':
         execfile(options.script, globals(), globals())
 
-    oOutput = cOutputResult(options.output)
+    oOutput = cOutputResult(options)
     for filename in filenames:
+        oOutput.Filename(filename)
         PythonPerLineSingle(expression, filename, oOutput, options)
     oOutput.Close()
 
@@ -373,7 +455,7 @@ def Main():
 
     oParser = optparse.OptionParser(usage='usage: %prog [options] expression [[@]file ...]\n' + __description__ + moredesc, version='%prog ' + __version__)
     oParser.add_option('-m', '--man', action='store_true', default=False, help='Print manual')
-    oParser.add_option('-o', '--output', type=str, default='', help='Output to file')
+    oParser.add_option('-o', '--output', type=str, default='', help='Output to file (# supported)')
     oParser.add_option('-s', '--script', type=str, default='', help='Script with definitions to include')
     oParser.add_option('-e', '--execute', default='', help='Commands to execute')
     oParser.add_option('-r', '--range', type=str, default='', help='Parameters to generate input with xrange')