Update settings import logging, Fix backend/frontend restart during S…

…ettings Import

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## 8.15.5 (Unreleased)
 
+### Bugfixes
+
+ - Fix backend/frontend restart during Settings Import
+
 ### Features
 
  - Add listener and do_not_run_periodically as Function module options

mycodo/mycodo_flask/routes_page.py

@@ -469,7 +469,8 @@ def page_export():
                       'as any unmet dependencies will be installed. '
                       'Additionally, the daemon will be stopped during this '
                       'process and all users will be logged out at the '
-                      'completion of the import.', 'success')
+                      'completion of the import. View the Settings Import '
+                      'Log for details.', 'success')
                 return redirect(url_for('routes_authentication.logout'))
             else:
                 flash('An error occurred during the settings database import.',

mycodo/mycodo_flask/utils/utils_export.py

@@ -30,6 +30,7 @@
 from mycodo.utils.system_pi import assure_path_exists, cmd_output
 from mycodo.utils.tools import (create_measurements_export,
                                 create_settings_export)
+from mycodo.utils.utils import append_to_log
 from mycodo.utils.widget_generate_html import generate_widget_html
 
 logger = logging.getLogger(__name__)
@@ -67,7 +68,7 @@ def export_measurements(form):
                     start=start_seconds, end=end_seconds)
                 return url
         except Exception as err:
-            error.append("Error: {}".format(err))
+            error.append(f"Error: {err}")
     else:
         flash_form_errors(form)
         return
@@ -101,7 +102,7 @@ def export_settings():
         else:
             error.append(data)
     except Exception as err:
-        error.append("Error: {}".format(err))
+        error.append(f"Error: {err}")
 
     flash_success_errors(error, action, url_for('routes_page.page_export'))
 
@@ -135,7 +136,7 @@ def export_influxdb():
         else:
             error.append("Could not determine Influxdb host/version")
     except Exception as err:
-        error.append("Error: {}".format(err))
+        error.append(f"Error: {err}")
 
     flash_success_errors(error, action, url_for('routes_page.page_export'))
 
@@ -145,37 +146,53 @@ def export_influxdb():
 #
 
 def thread_import_settings(tmp_folder):
-    logger.info("Finishing up settings import")
+    logger.info("Finishing up settings import with thread_import_settings()")
 
     try:
+        # Initialize
+        cmd = f"{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper initialize | ts '[%Y-%m-%d %H:%M:%S]' >> {IMPORT_LOG_FILE} 2>&1"
+        _, _, _ = cmd_output(cmd)
+
         # Upgrade database
+        append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Upgrading database\n")
         cmd = f"{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper upgrade_database | ts '[%Y-%m-%d %H:%M:%S]' >> {IMPORT_LOG_FILE} 2>&1"
         _, _, _ = cmd_output(cmd)
 
         # Install/update dependencies (could take a while)
+        append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Installing dependencies (this can take a while)...\n")
         cmd = f"{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper update_dependencies | ts '[%Y-%m-%d %H:%M:%S]' >> {IMPORT_LOG_FILE} 2>&1"
         _, _, _ = cmd_output(cmd)
 
+        # Generate widget HTML
+        generate_widget_html()
+
         # Initialize
         cmd = f"{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper initialize | ts '[%Y-%m-%d %H:%M:%S]' >> {IMPORT_LOG_FILE} 2>&1"
         _, _, _ = cmd_output(cmd)
 
-        # Generate widget HTML
-        generate_widget_html()
-
+        # Start Mycodo daemon (backend)
+        append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Restarting backend")
         if DOCKER_CONTAINER:
             subprocess.Popen('docker start mycodo_daemon 2>&1', shell=True)
         else:
-            # Start Mycodo daemon (backend)
-            cmd = "{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper daemon_start | ts '[%Y-%m-%d %H:%M:%S]' >> {IMPORT_LOG_FILE} 2>&1"
-            _, _, _ = cmd_output(cmd)
+            cmd = f"{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper daemon_restart | ts '[%Y-%m-%d %H:%M:%S]' >> {IMPORT_LOG_FILE} 2>&1"
+            a, b, c = cmd_output(cmd)
 
         # Delete tmp directory if it exists
         if os.path.isdir(tmp_folder):
             shutil.rmtree(tmp_folder)
+
+        # Reload Mycodo Flask (frontend)
+        append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Reloading frontend")
+        if DOCKER_CONTAINER:
+            subprocess.Popen('docker start mycodo_flask 2>&1', shell=True)
+        else:
+            cmd = f"{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper frontend_reload | ts '[%Y-%m-%d %H:%M:%S]' >> {IMPORT_LOG_FILE} 2>&1"
+            _, _, _ = cmd_output(cmd)
     except:
         logger.exception("thread_import_settings()")
 
+    append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Settings Import complete")
     logger.info("Settings import complete")
 
 
@@ -191,7 +208,8 @@ def import_settings(form):
     error = []
 
     try:
-        logger.info("Beginning settings import")
+        logger.info("Beginning Settings Import")
+        append_to_log(IMPORT_LOG_FILE, f"\n\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Settings Import initiated")
         correct_format = 'Mycodo_MYCODOVERSION_Settings_DBVERSION_HOST_DATETIME.zip'
         upload_folder = os.path.join(INSTALL_DIRECTORY, 'upload')
         tmp_folder = os.path.join(upload_folder, 'mycodo_db_tmp')
@@ -217,36 +235,23 @@ def import_settings(form):
             # Compare the uploaded filename parts to the correct parts
             try:
                 if name_split[0] != correct_name_1:
-                    error.append(
-                        "Invalid file name: {n}: {fn} != {cn}.".format(
-                            n=file_name,
-                            fn=name_split[0],
-                            cn=correct_name_1))
-                    error.append("Correct format is: {fmt}".format(
-                        fmt=correct_format))
+                    error.append(f"Invalid file name: {file_name}: {name_split[0]} != {correct_name_1}.")
+                    error.append(f"Correct format is: {correct_format}")
                 elif name_split[2] != correct_name_2:
-                    error.append(
-                        "Invalid file name: {n}: {fn} != {cn}".format(
-                            n=file_name,
-                            fn=name_split[2],
-                            cn=correct_name_2))
-                    error.append("Correct format is: {fmt}".format(
-                        fmt=correct_format))
+                    error.append(f"Invalid file name: {file_name}: {name_split[2]} != {correct_name_2}")
+                    error.append(f"Correct format is: {correct_format}")
                 elif extension != correct_extension:
                     error.append("Extension not 'zip'")
                 elif name_split[1] > MYCODO_VERSION:
                     error.append(
-                        "Invalid Mycodo version: {fv} > {mv}. "
-                        "Only databases <= {mver} can only be imported".format(
-                            fv=name_split[1],
-                            mv=MYCODO_VERSION,
-                            mver=name_split[1]))
+                        f"Invalid Mycodo version: {name_split[1]} > {MYCODO_VERSION}. "
+                        f"Only databases <= {name_split[1]} can only be imported")
             except Exception as err:
-                error.append(
-                    "Exception while verifying file name: {err}".format(err=err))
+                error.append(f"Exception while verifying file name: {err}")
 
         if not error:
             logger.info("Saving import file")
+            append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Saving import file")
             # Save file to upload directory
             filename = secure_filename(
                 form.settings_import_file.data.filename)
@@ -260,32 +265,31 @@ def import_settings(form):
             try:
                 file_list = zipfile.ZipFile(full_path, 'r').namelist()
                 if DATABASE_NAME not in file_list:
-                    error.append("{} not found in zip: {}".format(
-                        DATABASE_NAME, ", ".join(file_list)))
+                    error.append(f"{DATABASE_NAME} not found in zip: {', '.join(file_list)}")
             except Exception as err:
-                error.append("Exception checking files in zip: "
-                             "{err}".format(err=err))
+                error.append(f"Exception checking files in zip: {err}")
 
         if not error:
             logger.info("Unzipping import file")
+            append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Unzipping import file")
             # Unzip file
             try:
                 assure_path_exists(tmp_folder)
                 zip_ref = zipfile.ZipFile(full_path, 'r')
                 zip_ref.extractall(tmp_folder)
                 zip_ref.close()
             except Exception as err:
-                error.append("Exception while extracting zip file: "
-                             "{err}".format(err=err))
+                error.append(f"Exception while extracting zip file: {err}")
 
         if not error:
             logger.info("Stopping daemon and copying files")
+            append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Stopping daemon and copying files")
             try:
                 if DOCKER_CONTAINER:
                     subprocess.Popen('docker stop mycodo_daemon 2>&1', shell=True)
                 else:
                     # Stop Mycodo daemon (backend)
-                    cmd = f"{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper daemon_stop | ts '[%Y-%m-%d %H:%M:%S]' >> {IMPORT_LOG_FILE} 2>&1"
+                    cmd = f"{INSTALL_DIRECTORY}/mycodo/scripts/mycodo_wrapper daemon_stop"
                     _, _, _ = cmd_output(cmd)
 
                 # Backup current database and replace with extracted mycodo.db
@@ -303,13 +307,14 @@ def import_settings(form):
                     PATH_INPUTS_CUSTOM,
                     PATH_OUTPUTS_CUSTOM,
                     PATH_WIDGETS_CUSTOM,
+                    PATH_USER_SCRIPTS,
                     PATH_HTML_USER,
-                    PATH_PYTHON_CODE_USER,
-                    PATH_USER_SCRIPTS
+                    PATH_PYTHON_CODE_USER
                 ]
 
                 # Delete custom functions/inputs/outputs/widgets and generated HTML/Python code
                 for each_dir in delete_directories:
+                    append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Deleting directory: {each_dir}")
                     if not os.path.exists(each_dir):
                         continue
                     for folder_name, sub_folders, filenames in os.walk(each_dir):
@@ -328,21 +333,26 @@ def import_settings(form):
                     (PATH_INPUTS_CUSTOM, "custom_inputs"),
                     (PATH_OUTPUTS_CUSTOM, "custom_outputs"),
                     (PATH_WIDGETS_CUSTOM, "custom_widgets"),
-                    (PATH_USER_SCRIPTS, "user_scripts")
+                    (PATH_USER_SCRIPTS, "user_scripts"),
+                    (PATH_HTML_USER, "user_html"),
+                    (PATH_PYTHON_CODE_USER, "user_python_code")
                 ]
 
                 # Restore zipped custom functions/inputs/outputs/widgets
                 for each_dir in restore_directories:
+                    append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Restoring {each_dir[1]} directory: {each_dir[0]}")
                     extract_dir = os.path.join(tmp_folder, each_dir[1])
                     if not os.path.exists(extract_dir):
                         continue
                     for folder_name, sub_folders, filenames in os.walk(extract_dir):
                         for filename in filenames:
                             file_path = os.path.join(folder_name, filename)
                             new_path = os.path.join(each_dir[0], filename)
+                            append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Restoring {new_path}")
                             try:
                                 shutil.move(file_path, new_path)
                             except:
+                                append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Error: Could not restore {filename}")
                                 logger.exception("Moving file")
 
                 logger.info("Finalizing import")
@@ -354,11 +364,15 @@ def import_settings(form):
                 return backup_name
             except Exception as err:
                 logger.exception("Settings import")
-                error.append("Exception while replacing database: "
-                             "{err}".format(err=err))
+                error.append(f"Exception while replacing database: {err}")
                 return None
 
     except Exception as err:
         error.append("Exception: {}".format(err))
 
+    if error:
+        append_to_log(IMPORT_LOG_FILE, f"\n\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Could not complete Settings Import, error(s) found:")
+    for each_err in error:
+        append_to_log(IMPORT_LOG_FILE, f"\n[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] Error: {each_err}")
+
     flash_success_errors(error, action, url_for('routes_page.page_export'))

mycodo/scripts/upgrade_commands.sh

@@ -153,6 +153,9 @@ case "${1:-''}" in
         if [[ ! -e /var/log/mycodo/mycododependency.log ]]; then
             touch /var/log/mycodo/mycododependency.log
         fi
+        if [[ ! -e /var/log/mycodo/mycodoimport.log ]]; then
+            touch /var/log/mycodo/mycodoimport.log
+        fi
         if [[ ! -e /var/log/mycodo/mycodoupgrade.log ]]; then
             touch /var/log/mycodo/mycodoupgrade.log
         fi
@@ -675,6 +678,9 @@ case "${1:-''}" in
         if [[ ! -e /var/log/mycodo/mycododependency.log ]]; then
             touch /var/log/mycodo/mycododependency.log
         fi
+        if [[ ! -e /var/log/mycodo/mycodoimport.log ]]; then
+            touch /var/log/mycodo/mycodoimport.log
+        fi
         if [[ ! -e /var/log/mycodo/mycodoupgrade.log ]]; then
             touch /var/log/mycodo/mycodoupgrade.log
         fi

mycodo/utils/tools.py

@@ -11,32 +11,23 @@
 
 from dateutil import relativedelta
 
-from mycodo.config import INSTALL_DIRECTORY
-from mycodo.config import PATH_FUNCTIONS_CUSTOM
-from mycodo.config import PATH_ACTIONS_CUSTOM
-from mycodo.config import PATH_INPUTS_CUSTOM
-from mycodo.config import PATH_OUTPUTS_CUSTOM
-from mycodo.config import PATH_USER_SCRIPTS
-from mycodo.config import PATH_WIDGETS_CUSTOM
-from mycodo.config import SQL_DATABASE_MYCODO
-from mycodo.config import USAGE_REPORTS_PATH
-from mycodo.databases.models import Conversion
-from mycodo.databases.models import DeviceMeasurements
-from mycodo.databases.models import EnergyUsage
-from mycodo.databases.models import Misc
-from mycodo.databases.models import Output
-from mycodo.databases.models import OutputChannel
+from mycodo.config import (INSTALL_DIRECTORY, PATH_ACTIONS_CUSTOM,
+                           PATH_FUNCTIONS_CUSTOM, PATH_HTML_USER,
+                           PATH_INPUTS_CUSTOM, PATH_OUTPUTS_CUSTOM,
+                           PATH_PYTHON_CODE_USER, PATH_USER_SCRIPTS,
+                           PATH_WIDGETS_CUSTOM, SQL_DATABASE_MYCODO,
+                           USAGE_REPORTS_PATH)
+from mycodo.databases.models import (Conversion, DeviceMeasurements,
+                                     EnergyUsage, Misc, Output, OutputChannel)
 from mycodo.utils.database import db_retrieve_table_daemon
-from mycodo.utils.influx import average_past_seconds
-from mycodo.utils.influx import average_start_end_seconds
-from mycodo.utils.influx import output_sec_on
+from mycodo.utils.influx import (average_past_seconds,
+                                 average_start_end_seconds, output_sec_on)
 from mycodo.utils.logging_utils import set_log_level
 from mycodo.utils.outputs import parse_output_information
-from mycodo.utils.system_pi import assure_path_exists
-from mycodo.utils.system_pi import cmd_output
-from mycodo.utils.system_pi import parse_custom_option_values_output_channels_json
-from mycodo.utils.system_pi import return_measurement_info
-from mycodo.utils.system_pi import set_user_grp
+from mycodo.utils.system_pi import (
+    assure_path_exists, cmd_output,
+    parse_custom_option_values_output_channels_json, return_measurement_info,
+    set_user_grp)
 
 logger = logging.getLogger("mycodo.tools")
 logger.setLevel(set_log_level(logging))
@@ -105,7 +96,9 @@ def create_settings_export(save_path=None):
                 (PATH_INPUTS_CUSTOM, "custom_inputs"),
                 (PATH_OUTPUTS_CUSTOM, "custom_outputs"),
                 (PATH_WIDGETS_CUSTOM, "custom_widgets"),
-                (PATH_USER_SCRIPTS, "user_scripts")
+                (PATH_USER_SCRIPTS, "user_scripts"),
+                (PATH_HTML_USER, "user_html"),
+                (PATH_PYTHON_CODE_USER, "user_python_code")
             ]
             for each_backup in export_directories:
                 if not os.path.exists(each_backup[0]):

mycodo/utils/utils.py

@@ -21,11 +21,28 @@
 #
 #  Contact at kylegabriel.com
 
+import logging
 import random
+import os
 import re
 import string
 import sys
 
+from mycodo.utils.logging_utils import set_log_level
+
+logger = logging.getLogger("mycodo.utils")
+logger.setLevel(set_log_level(logging))
+
+
+def append_to_log(log_file, str_append):
+    """Write to a file. Do not use when may be executed more than once at a time."""
+    if os.path.exists:
+        w_type = "a"
+    else:
+        w_type = "w"
+    with open(log_file, w_type) as f:
+        f.write(str_append)
+
 
 def is_email(email):
     """