Widespread changes, bugfixes; now supporting backfilling of missing datetime values

Author: nabelekt

README.md

@@ -6,31 +6,38 @@ Your input file should have a format similiar to those under `test_data`. To hav
 
 If working with a database, the database access information/credentials must be setup in `db_creds.py`.
 
+The `cycle_start` `talker`+`sentence_type`, e.g. `GNRMC`, passed to `datetime_stamp_sentences()` and `assign_cycle_ids()`, must appear once and only once in each cycle, and it must be at the beginning of each cycle. For sentences to be datetime stamped, `cycle_start` sentences must contain date and time information. The `--backfill_datetimes` flag can be used to back fill datetimes for cycles where that information was not avaiable.
+
 
 ## Usage
 ```
-$ cd ~/Downloads/nmea_parser/
-$ pip install -r requirements.txt
+$ cd nmea_data_convert/
+$ pip install -r requirements.txt 
 ...
-$ python nmea_parser.py --help
-usage: nmea_parser.py [-h] [--drop_previous_db_tables] filepath {csv,db,both}
+$ python nmea_data_convert.py --help
+usage: nmea_data_convert.py [-h] [--drop_previous_db_tables]
+                            [--backfill_datetimes]
+                            filepath {csv,db,both}
 
 positional arguments:
   filepath              file system path to file containing NMEA data
   {csv,db,both}         where to output data: CSV files, database, or both
 
 optional arguments:
   -h, --help            show this help message and exit
-  --drop_previous_db_tables
+  --drop_previous_db_tables, --dropt
                         drop previous DB tables before importing new data;
                         only applies when output_method is 'db' or 'both'
+  --backfill_datetimes, --bfdt
+                        backfill datetimes where missing by extrapolating from
+                        messages with datetime information
 ```
 ## Examples
 ### Example 1
 ```
 $ ls -l *.csv
 ls: *.csv: No such file or directory
-$ python nmea_parser.py test_data/test_data_all.nmea csv
+$ python nmea_data_convert.py test_data/test_data_all.nmea csv
 
 Reading in data... done.
 
@@ -61,7 +68,7 @@ $ ls -l *.csv
 
 ### Example 2
 ```
-$ python nmea_parser.py test_data/test_data_all.nmea db
+$ python nmea_data_convert.py test_data/test_data_all.nmea db
 
 Reading in data... done.
 
@@ -82,7 +89,7 @@ All done. Exiting.
 
 ### Example 3
 ```
-$ python nmea_parser.py test_data/test_data_all.nmea both
+$ python nmea_data_convert.py test_data/test_data_all.nmea both --bfdt --dropt
 
 Reading in data... done.
 
@@ -98,6 +105,14 @@ Writing data to CSVs... data from logfile 'test_data/test_data_all.nmea' written
   test_data_all_GNGLL.csv
 done.
 
+Dropping database table nmea_gl_gsv (and any dependent objects) if it exists.
+Dropping database table nmea_gn_gga (and any dependent objects) if it exists.
+Dropping database table nmea_gn_gll (and any dependent objects) if it exists.
+Dropping database table nmea_gn_gsa (and any dependent objects) if it exists.
+Dropping database table nmea_gn_rmc (and any dependent objects) if it exists.
+Dropping database table nmea_gn_vtg (and any dependent objects) if it exists.
+Dropping database table nmea_gp_gsv (and any dependent objects) if it exists.
+
 Writing data to database... data from logfile 'test_data/test_data_all.nmea' written to:
   'nmea_gn_rmc' table in 'nmea_data' database
   'nmea_gn_vtg' table in 'nmea_data' database
@@ -119,4 +134,14 @@ https://www.trimble.com/OEM_ReceiverHelp/V4.44/en/NMEA-0183messages_MessageOverv
 
 https://www.u-blox.com/sites/default/files/products/documents/u-blox8-M8_ReceiverDescrProtSpec_%28UBX-13003221%29.pdf (section 31 'NMEA Protocol')
 
-https://www.sparkfun.com/datasheets/GPS/NMEA%20Reference%20Manual1.pdf
+https://www.sparkfun.com/datasheets/GPS/NMEA%20Reference%20Manual1.pdf
+
+https://www.nmea.org/Assets/20190303%20nmea%200183%20talker%20identifier%20mnemonics.pdf
+
+
+## Support
+If you find this tool useful, please consider supporting development of this tool and other tools like it. You can do so using the `Sponsor` button at the top of the [GitHub page](https://github.com/Petrichor-Labs/nmea_data_convert).
+
+
+## Discussion
+For any questions, feedback, or other discussion items, please feel free to post in the [`Discussions` tab on the GitHub page](https://github.com/Petrichor-Labs/nmea_data_convert/discussions).

db_data_import.py

@@ -29,7 +29,7 @@ def send_data_to_db(log_file_path, dfs, table_name_base, table_name_suffixes=Non
             table_name = table_name + '_' + table_name_suffixes[df_idx]
 
         try:
-            df.to_sql(table_name, engine, method='multi', if_exists=if_exists_opt_loc)
+            df.to_sql(table_name, engine, method='multi', if_exists=if_exists_opt_loc, index=False)
         except (sqlalchemy.exc.OperationalError, psycopg2.OperationalError) as e:
             sys.exit(f"\n\n\033[1m\033[91mERROR writing to database:\n  {e}\033[0m\n\nExiting.\n\n")  # Print error text bold and red
 

db_utils.py

@@ -1,5 +1,7 @@
+import sys
 import psycopg2
 from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT
+import sqlalchemy
 import functools
 print = functools.partial(print, flush=True)  # Prevent print statements from buffering till end of execution
 
@@ -61,8 +63,12 @@ def setup_db_connection():
 
     db_access_str = f'postgresql://{db_creds.DB_USER}:{db_creds.DB_PASSWORD}@{db_creds.DB_HOST}:{db_creds.DB_PORT}/{db_creds.DB_NAME}'
 
-    # Start a PostgreSQL database session   
-    psqlCon = psycopg2.connect(db_access_str);
+    # Start a PostgreSQL database session
+    try:
+        psqlCon = psycopg2.connect(db_access_str);
+    except (sqlalchemy.exc.OperationalError, psycopg2.OperationalError) as e:
+        sys.exit(f"\n\033[1m\033[91mERROR connecting to database:\n  {e}\033[0m\n\nExiting.\n\n")  # Print error text bold and red
+    
     psqlCon.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT);
 
     # Open a database cursor