Re did all the plugin + changes on a new branch since my old one was having conflicts and could not be merged. #366

Author: cleaaum

sarra/plugins/html_page.py

@@ -30,7 +30,7 @@ def file_size_fix(self):
                self.mysize = "%d" % isize
 
         except:
-               self.logger.debug("bad size %s" % self.mysize)
+               # self.logger.debug("bad size %s" % self.mysize)
                return
 
         return

sarra/plugins/line_date.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""comments:
+    This plugin modifies the date of parent.line to a standard date fomrat: year-month-date hours:minutes:seconds
+    An example of this date format is : 2021-09-06 11:09:00, which is Septemebr 6th, 2021 11:09PM
+    Ten formats are acceptd so far, more can be added if needed (format on https://strftime.org/ ).
+    FIXME: french input like Fev will not work - only Feb is accepted for the month
+    If year is not provided, this means that the file is < 6 months old, so depending on todays date, assign
+    appropriate year (for todays year: jan-jun -> assign prev year, for jul-dec assign current year)
+    Note: is it possible for a file to be more than 6 months old and have the format Mo Day TIME ? (problematic)
+"""
+class Line_date(object):
+    def __init__(self, parent):
+        pass
+
+    def normalize_date_format(self, parent):
+        import dateparser
+        line_split = parent.line.split()
+        # specify input for this routine, line format could change
+        # line_mode.py format "-rwxrwxr-x 1 1000 1000 8123 24 Mar 22:54 2017-03-25-0254-CL2D-AUTO-minute-swob.xml"
+        file_date = line_split[5] + " " + line_split[6] + " " + line_split[7]
+        # case 1: the date contains '-' implies the date is in 1 string not 3 seperate ones, and H:M is also provided
+        if "-" in file_date: file_date = line_split[5] + " " + line_split[6]
+        current_date = datetime.datetime.now()
+        standard_date_format = dateparser.parse(file_date,
+                                                settings={
+                                                    'RELATIVE_BASE': datetime.datetime(1900, 1, 1),
+                                                    'TIMEZONE': parent.destination_timezone,
+                                                    'TO_TIMEZONE': 'UTC'})
+        if standard_date_format is not None:
+            type(standard_date_format)
+            # case 2: the year was not given, it is defaulted to 1900. Must find which year (this one or last one).
+            if standard_date_format.year == 1900:
+                if standard_date_format.month - current_date.month >= 6:
+                    standard_date_format = standard_date_format.replace(year=(current_date.year - 1))
+                else:
+                    standard_date_format = standard_date_format.replace(year=current_date.year)
+            parent.logger.debug("Oldline is: " + parent.line)
+            parent.line = parent.line.replace(file_date, str(standard_date_format))
+            parent.logger.debug("Newline is: " + parent.line)
+        return
+
+    def perform(self,parent):
+        if hasattr(parent, 'line'):
+            self.normalize_date_format(parent)
+        return True
+
+line_date = Line_date(self)
+self.on_line = line_date.perform

sarra/sr_config.py

@@ -659,6 +659,7 @@ def declare_option(self,option):
     def defaults(self):
         self.logger.debug("sr_config defaults")
         self.file_time_limit      = self.duration_from_str("60d")
+        self.destination_timezone = 'UTC'
         self.retry_mode           = True
         self.retry_ttl            = None
 
@@ -903,6 +904,7 @@ def defaults(self):
 
         #self.on_post_list = [ self.on_post ]
         self.execfile("on_line",'line_mode')
+        self.execfile("on_line", 'line_date')
 
 
     # this function converts duration into a specifid unit: [milliseconds, seconds or days]
@@ -1594,6 +1596,8 @@ def option(self,words):
                      self.logger.debug("Masks %s"% self.masks)
                 elif words0 =='file_time_limit':
                     self.file_time_limit = self.duration_from_str(words1)
+                elif words0 == 'destination_timezone':
+                    self.destination_timezone = words1
                 elif words0 in ['accept_unmatched','accept_unmatch','au']: # See: sr_config.7
                      if (words1 is None) or words[0][0:1] == '-' : 
                         self.accept_unmatch = True

sarra/sr_poll.py

@@ -68,7 +68,7 @@
 
 import os, sys, time
 import datetime
-
+from dateparser import parse
 # ============================================================
 # DECLARE TRICK for false self.poster
 
@@ -144,55 +144,6 @@ def check(self):
             self.pulls[maskDir].append(mask)
 
 
-    def _file_date_exceed_limit(self, date, time_limit):
-        """comments:
-            This method compares today's date to the file's date by creating a date time object
-            Four formats are acceptd so far, more can be added if needed (format on https://strftime.org/ )
-            Files with future dates are processed as long as the (future date - todays date) is < time_limit.
-            FIXME: french input like Fev will not work - only Feb is accepted for the month
-            If year is not provided, this means that the file is < 6 months old, so depending on todays date,
-                assign appropriate year (for jan-jun -> assign prev year, for jul-dec assign current year)
-            Note: is it possible for a file to be more than 6 months old and have the format Mo Day TIME ? (problematic)
-        """
-        time_limit=int(time_limit)
-        current_date = datetime.datetime.now()
-        try:
-            date_temp = datetime.datetime.strptime(date, '%d %b %H:%M')
-            if date_temp.month - current_date.month >= 6:
-                file_date = date_temp.replace(year=(current_date.year - 1))
-            else:
-                file_date = date_temp.replace(year=current_date.year)
-            self.logger.debug("File date is: " + str(file_date) +
-                             " > File is " + str(abs((file_date - current_date).seconds))+" seconds old")
-            return abs((file_date - current_date).seconds) < time_limit
-        except Exception as e:
-            try:
-                date_temp = datetime.datetime.strptime(date, '%b %d %H:%M')
-                if date_temp.month - current_date.month >= 6:
-                    file_date = date_temp.replace(year=(current_date.year - 1))
-                else:
-                    file_date = date_temp.replace(year=current_date.year)
-                self.logger.debug("File date is: " + str(file_date) +
-                                 " > File is " + str(abs((file_date - current_date).seconds))+" seconds old")
-                return abs((file_date - current_date).seconds) < time_limit
-            except Exception as e:
-                try:
-                    file_date = datetime.datetime.strptime(date, '%b %d %Y')
-                    self.logger.debug("File date is: " + str(file_date) +
-                                     " > File is " + str(abs((file_date - current_date).seconds)) + " seconds old")
-                    return abs((file_date - current_date).seconds) < time_limit
-                except Exception as e:
-                    try:
-                        file_date = datetime.datetime.strptime(date, '%d %b %Y')
-                        self.logger.debug("File date is: " + str(file_date) + " > File is " +
-                                          str(abs((file_date - current_date).seconds)) + " seconds old")
-                        return abs((file_date - current_date).seconds) < time_limit
-                    except Exception as e:
-                        warning_msg = str(e)
-                        # self.logger.error("%s, assuming ok" % warning_msg)
-                        self.logger.error("Assuming ok, unrecognized date format, %s" % date)
-                        return True
-
     # find differences between current ls and last ls
     # only the newer or modified files will be kept...
 
@@ -207,6 +158,7 @@ def differ_ls_file(self, ls, lspath):
 
         old_ls = self.load_ls_file(lspath)
 
+        file_within_date_limit = True
         # compare
 
         filelst = []
@@ -215,34 +167,33 @@ def differ_ls_file(self, ls, lspath):
         for f in new_lst:
             # self.logger.debug("checking %s (%s)" % (f,ls[f]))
             try:
-                str1 = ls[f]
-                str2 = str1.split()
-                # specify input for this routine.
-                # ls[f] format controlled by online plugin (line_mode.py)
-                # this format could change depending on plugin
-                # line_mode.py format "-rwxrwxr-x 1 1000 1000 8123 24 Mar 22:54 2017-03-25-0254-CL2D-AUTO-minute-swob.xml"
-                date = str2[5] + " " + str2[6] + " " + str2[7]
-                if self._file_date_exceed_limit(date, self.file_time_limit):
-                    self.logger.debug("File should be processed")
-                    # execute rest of code
-                    # keep a newer entry
-                    if not f in old_ls:
-                        # self.logger.debug("IS NEW %s" % f)
-                        filelst.append(f)
-                        desclst[f] = ls[f]
-                        continue
-
-                    # keep a modified entry
-                    if ls[f] != old_ls[f]:
-                        # self.logger.debug("IS DIFFERENT %s from (%s,%s)" % (f,old_ls[f],ls[f]))
-                        filelst.append(f)
-                        desclst[f] = ls[f]
-                        continue
-                else:
-                    self.logger.debug("File should be skipped")
-                    # ignore rest of code and re iterate
+                line_split = ls[f].split()
+                date = line_split[5] + " " + line_split[6]
+                file_date = parse(date)
+                current_date = datetime.datetime.now()
+                file_within_date_limit = abs((file_date - current_date).seconds) < self.file_time_limit
             except:
+                self.logger.error("Assuming ok, incorrect date format for line:  %s" % ls[f])
                 pass
+            if file_within_date_limit:
+                self.logger.debug("File should be processed")
+                # execute rest of code
+                # keep a newer entry
+                if not f in old_ls:
+                    # self.logger.debug("IS NEW %s" % f)
+                    filelst.append(f)
+                    desclst[f] = ls[f]
+                    continue
+
+                # keep a modified entry
+                if ls[f] != old_ls[f]:
+                    # self.logger.debug("IS DIFFERENT %s from (%s,%s)" % (f,old_ls[f],ls[f]))
+                    filelst.append(f)
+                    desclst[f] = ls[f]
+                    continue
+            else:
+                self.logger.debug("File should be skipped")
+                # ignore rest of code and re iterate
             # self.logger.debug("IS IDENTICAL %s" % f)
 
         return filelst, desclst

setup.py

@@ -72,6 +72,7 @@ def read(*parts):
     "watchdog", 
     "netifaces", 
     "humanize",
-    "psutil" ]
+    "psutil",
+    "dateparser"]
 
 )