Fixed some bugs with KML. Added data samples. Added stats for # of co…

…rrupt packets, unknown packets, non-dat files

DROP.py

@@ -25,7 +25,7 @@
 parser.add_argument('-t', help='path to input Flight Record CSV file or directory')
 parser.add_argument('-f', '--force', help='force processing of file(s) if correct file header is not found', action='store_true')
 parser.add_argument('-k', '--kml', help='Provide the output KML file path/name if you wish to create a KML file.')
-parser.add_argument('-s', '--kmlscale', help='Set the point scale of the kml file. i.e. -s 2 = 1 kml point for every 2 real points.')
+parser.add_argument('-s', '--kmlscale', help='Set the point scale of the kml file. i.e. -s 2 = 1 kml point for every 2 real points. Default is 1:1.')
 ################################################# Custom Exceptions (Put in a seperate file later)
 
 class NotDATFileError(Exception):
@@ -83,17 +83,21 @@ def __str__(self):
     print('*** WARNING: The FORCE flag has been set. ALL files will be processed (not just standard DJI DAT files). ***')
 
 kmlFile = args.kml
-if kmlFile == None:
+kmlScale = 1
+if kmlFile != None:
+    print('kmlFile is: ' + str(kmlFile))
     spl_path = os.path.split(in_arg)
     kmlFile = spl_path[len(spl_path)-1].split('.')[0] + '-Map.kml'
-print("output to KML file: " + str(kmlFile))
-
-if args.kmlscale != None:
-    try:
-        kmlScale = int(args.kmlscale)
-    except:
-        print('Error: scale must be whole number integer.')
-        exit()
+    print("output to KML file: " + str(kmlFile))
+
+    if args.kmlscale != None:
+        try:
+            kmlScale = int(args.kmlscale)
+        except:
+            print('Error: scale must be whole number integer.')
+            exit()
+    else:
+        kmlScale = 1
 
 ################################################# Process TXT files
 
@@ -102,6 +106,7 @@ def __str__(self):
     txtfiles = ProcessFRCSV(in_tf)
 
 #################################################
+nonDatFiles = 0   # Keeps track of the number of non-DAT files that DROP attempted to process
 
 for ifn in in_files_list:
     if os.path.isdir(out_path):
@@ -133,6 +138,7 @@ def __str__(self):
         #print(b"BUILD".decode('ascii'))
         if build.decode('ascii') != b"BUILD".decode('ascii'):
             if not force:
+                nonDatFiles += 1
                 raise NotDATFileError(in_fn)
             else:
                 print('*** WARNING: ' + in_fn + ' is not a recognized DJI DAT file but will be processed anyway because the FORCE flag was set. ***')
@@ -182,6 +188,9 @@ def __str__(self):
             alternateStructure = True
         message = None
         message = Message(meta, kmlFile, kmlScale)      # create a new, empty message
+
+        corruptPackets = 0   # keeps track of the number of corrupt packets - data blocks that do not meet the minimum formatting requirements to be a DJI flight data packet
+        unknownPackets = 0   # keeps track of the number of unrecognized packets - packets that are of the DJI flight data format but we do not know how to parse the payload
 
         start_issue = True
         while len(byte) != 0:
@@ -225,11 +234,14 @@ def __str__(self):
 
                     message.writeRow(writer, thisPacketTickNo)
 
-                    message.addPacket(pktlen, header, payload)
+                    if message.addPacket(pktlen, header, payload) == False:
+                        unknownPackets += 1
+
                     byte = in_file.read(1)
                 else:
                     byte = padding
             except CorruptPacketError as e:
+                corruptPackets += 1
                 print(e.value)
             except NoNewPacketError as e:
                 if start_issue:     # first time around the loop with this problem
@@ -250,7 +262,10 @@ def __str__(self):
         log_file.write('Command Used: ' + ' '.join(sys.argv) + '\n')
         log_file.write('Input File Name: ' + in_fn + '\n')
         log_file.write('Output File Name: ' + out_fn + '\n')
+        log_file.write('Number of Non-DAT Files: ' + str(nonDatFiles) + '\n')
         log_file.write('Number of Records Processed: ' + str(message.packetNum) + '\n')
+        log_file.write('Number of Corrupt Records: ' + str(corruptPackets) + '\n')
+        log_file.write('Number of Unrecognized Records: ' + str(unknownPackets) + '\n')
         #log_file.write('File Access Rights: ' + str(meta.st_mode) + '\n')
         #log_file.write('I-node: ' + str(meta.st_ino) + '\n')
         #log_file.write('Device Number: ' + str(meta.st_dev) + '\n')

modules/Message.py

@@ -64,7 +64,7 @@ def __init__(self, meta, kmlFile=None, kmlScale=1):
         self.kmlWriter = None
         if self.kmlFile != None:
             self.kmlWriter = simplekml.Kml()
-        self.kml_res = kmlScale
+            self.kml_res = kmlScale
         # *********************************************
         # **** WARNING: THIS IS NOT THE PREFERED METHOD TO OBTAIN THE START TIME
         # ctime is the time the meta data (permissions) were last changed
@@ -93,7 +93,8 @@ def addPacket(self, pktlen, header, payload):
                 if self.row_out.get('latitude') and self.row_out.get('longitude') and self.row_out.get('time(millisecond)'):
                     if self.row_out['latitude'] != '' and self.row_out['longitude'] != '' and self.row_out['time(millisecond)'] != '':
                         self.gps_fr_dict[self.row_out['time(millisecond)']] = [self.row_out['latitude'], self.row_out['longitude'], self.row_out.get('baroAlt', ''), self.row_out.get('satnum', ''), self.row_out.get('totalVolts', ''), self.row_out.get('flyc_state', '')]
-            #self.packets.append(packet)
+            return True
+        return False
 
     def getRow(self):
         if self.tickNo != None:
@@ -105,7 +106,7 @@ def getRow(self):
         return dict(self.row_out, **{'messageid':self.tickNo, 'offsetTime':offsetTime, 'logDateTime':logDateTime})
 
     def writeKml(self, row):
-        if self.kmlWriter:
+        if self.kmlWriter != None:
             if row.get('latitude', False) and row.get('longitude', False):
                 if self.point_cnt % self.kml_res == 0:
                     self.kmlWriter.newpoint(name=str(row.get('messageid')), coords=[(row.get('longitude'), row.get('latitude'))])    
@@ -126,5 +127,5 @@ def outToFile(self, fname):
                     of.write(str(d) + ': ' + str(self.gps_fr_dict[d]) + '\n')
 
     def finalizeKml(self):
-        if self.kmlWriter and self.kmlFile:
+        if self.kmlWriter != None and self.kmlFile != None:
             self.kmlWriter.save(self.kmlFile)