Merge pull request #1663 from YunoHost/new-log-streaming-api

POC for new log streaming API using a zero-mq broker

.gitlab/ci/test.gitlab-ci.yml

@@ -1,4 +1,7 @@
 .install_debs: &install_debs
+    # Temporary hack for the CI to install python3-zmq not yet in the image
+    - DEBIAN_FRONTEND=noninteractive apt update
+    - DEBIAN_FRONTEND=noninteractive apt --assume-yes install python3-zmq 
     - DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes apt --assume-yes -o Dpkg::Options::="--force-confold" --allow-downgrades install ${CI_PROJECT_DIR}/*.deb
 
 .test-stage:

debian/control

@@ -17,6 +17,7 @@ Depends: python3-all (>= 3.11),
  , python3-ldap, python3-zeroconf (>= 0.47), python3-lexicon,
  , python3-cryptography, python3-jwt, python3-passlib, python3-magic
  , python-is-python3, python3-pydantic, python3-email-validator
+ , python3-zmq
  , nginx, nginx-extras (>=1.22)
  , apt, apt-transport-https, apt-utils, aptitude, dirmngr
  , openssh-server, iptables, fail2ban, bind9-dnsutils

locales/en.json

@@ -562,14 +562,15 @@
     "log_backup_restore_app": "Restore '{}' from a backup archive",
     "log_backup_restore_system": "Restore system from a backup archive",
     "log_corrupted_md_file": "The YAML metadata file associated with logs is damaged: '{md_file}\nError: {error}'",
+    "log_diagnosis_run": "Run diagnosis",
     "log_does_exists": "There is no operation log with the name '{log}', use 'yunohost log list' to see all available operation logs",
-    "log_domain_add": "Add '{}' domain into system configuration",
+    "log_domain_add": "Add domain '{}'",
     "log_domain_config_set": "Update configuration for domain '{}'",
     "log_domain_dns_push": "Push DNS records for domain '{}'",
     "log_domain_main_domain": "Make '{}' the main domain",
-    "log_domain_remove": "Remove '{}' domain from system configuration",
-    "log_dyndns_subscribe": "Subscribe to a YunoHost subdomain '{}'",
-    "log_dyndns_unsubscribe": "Unsubscribe to a YunoHost subdomain '{}'",
+    "log_domain_remove": "Remove domain '{}'",
+    "log_dyndns_subscribe": "Register YunoHost subdomain '{}'",
+    "log_dyndns_unsubscribe": "Unregister YunoHost subdomain '{}'",
     "log_dyndns_update": "Update the IP associated with your YunoHost subdomain '{}'",
     "log_help_to_get_failed_log": "The operation '{desc}' could not be completed. Please share the full log of this operation using the command 'yunohost log share {name}' to get help",
     "log_help_to_get_log": "To view the log of the operation '{desc}', use the command 'yunohost log show {name}'",
@@ -592,6 +593,7 @@
     "log_tools_postinstall": "Postinstall your YunoHost server",
     "log_tools_reboot": "Reboot your server",
     "log_tools_shutdown": "Shutdown your server",
+    "log_tools_update": "Fetching available system updates and refreshing app catalog",
     "log_tools_upgrade": "Upgrade system packages",
     "log_user_create": "Add '{}' user",
     "log_user_delete": "Delete '{}' user",

src/__init__.py

@@ -140,6 +140,8 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun
     if not os.path.isdir(logdir):
         os.makedirs(logdir, 0o750)
 
+    base_handlers = ["file"] + (["cli"] if interface == "cli" else [])
+
     logging_configuration = {
         "version": 1,
         "disable_existing_loggers": True,
@@ -157,35 +159,28 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun
                 "class": "moulinette.interfaces.cli.TTYHandler",
                 "formatter": "tty-debug" if debug else "",
             },
-            "api": {
-                "level": "DEBUG" if debug else "INFO",
-                "class": "moulinette.interfaces.api.APIQueueHandler",
-            },
-            "portalapi": {
-                "level": "DEBUG" if debug else "INFO",
-                "class": "moulinette.interfaces.api.APIQueueHandler",
-            },
             "file": {
                 "class": "logging.FileHandler",
                 "formatter": "precise",
                 "filename": logfile,
             },
+
         },
         "loggers": {
             "yunohost": {
                 "level": "DEBUG",
-                "handlers": ["file", interface] if not quiet else ["file"],
+                "handlers": base_handlers if not quiet else ["file"],
                 "propagate": False,
             },
             "moulinette": {
                 "level": "DEBUG",
-                "handlers": ["file", interface] if not quiet else ["file"],
+                "handlers": base_handlers if not quiet else ["file"],
                 "propagate": False,
             },
         },
         "root": {
             "level": "DEBUG",
-            "handlers": ["file", interface] if debug else ["file"],
+            "handlers": base_handlers if debug else ["file"],
         },
     }
 
@@ -206,4 +201,6 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun
             logging_configuration["loggers"]["moulinette"]["handlers"].append("cli")
             logging_configuration["root"]["handlers"].append("cli")
 
+        from yunohost.utils.sse import start_log_broker
+        start_log_broker()
         configure_logging(logging_configuration)

src/app.py

@@ -1815,6 +1815,7 @@ def app_ssowatconf():
     logger.debug(m18n.n("ssowat_conf_generated"))
 
 
+@is_unit_operation(flash=True)
 def app_change_label(app, new_label):
     from yunohost.permission import user_permission_update
 
@@ -3252,6 +3253,7 @@ def _assert_system_is_sane_for_app(manifest, when):
             raise YunohostError("this_action_broke_dpkg")
 
 
+@is_unit_operation(flash=True)
 def app_dismiss_notification(app, name):
     assert isinstance(name, str)
     name = name.lower()

src/backup.py

@@ -2556,6 +2556,7 @@ def backup_info(name, with_details=False, human_readable=False):
     return result
 
 
+@is_unit_operation(flash=True)
 def backup_delete(name):
     """
     Delete a backup

src/diagnosis.py

@@ -34,6 +34,7 @@
 )
 
 from yunohost.utils.error import YunohostError, YunohostValidationError
+from yunohost.log import is_unit_operation
 
 logger = getLogger("yunohost.diagnosis")
 
@@ -154,8 +155,9 @@ def _dump_human_readable_reports(reports):
     return output
 
 
+@is_unit_operation(sse_only=True)
 def diagnosis_run(
-    categories=[], force=False, except_if_never_ran_yet=False, email=False
+    operation_logger, categories=[], force=False, except_if_never_ran_yet=False, email=False
 ):
     if (email or except_if_never_ran_yet) and not os.path.exists(DIAGNOSIS_CACHE):
         return
@@ -173,6 +175,8 @@ def diagnosis_run(
                 "diagnosis_unknown_categories", categories=", ".join(unknown_categories)
             )
 
+    operation_logger.start()
+
     issues = []
     # Call the hook ...
     diagnosed_categories = []
@@ -210,10 +214,12 @@ def diagnosis_run(
         logger.warning(m18n.n("diagnosis_display_tip"))
 
 
+@is_unit_operation(flash=True)
 def diagnosis_ignore(filter, list=False):
     return _diagnosis_ignore(add_filter=filter, list=list)
 
 
+@is_unit_operation(flash=True)
 def diagnosis_unignore(filter):
     return _diagnosis_ignore(remove_filter=filter)
 

src/dyndns.py

@@ -288,6 +288,7 @@ def dyndns_unsubscribe(operation_logger, domain, recovery_password=None):
     logger.success(m18n.n("dyndns_unsubscribed"))
 
 
+@is_unit_operation(flash=True)
 def dyndns_set_recovery_password(domain, recovery_password):
     keys = glob.glob(f"/etc/yunohost/dyndns/K{domain}.+*.key")