From 0a756f8fd8f29d3e87cef58ecc6431734da22ae1 Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sat, 26 Aug 2023 19:28:36 -0400 Subject: [PATCH] Add watchdog and watch roms --- backend/config/__init__.py | 2 +- backend/endpoints/scan.py | 4 +- backend/tasks/scan_library.py | 8 +-- backend/watcher.py | 87 ++++++++++++++++++++++++++++++ docker/init_scripts/init | 3 ++ docker/init_scripts/init_scheduler | 2 +- docker/init_scripts/init_watchdog | 4 ++ docker/init_scripts/init_worker | 2 +- poetry.lock | 41 +++++++++++++- pyproject.toml | 1 + 10 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 backend/watcher.py create mode 100644 docker/init_scripts/init_watchdog diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 5b18a48f6..9a3f3b48b 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -70,7 +70,7 @@ os.environ.get("ENABLE_RESCAN_ON_FILESYSTEM_CHANGE", "false") == "true" ) RESCAN_ON_FILESYSTEM_CHANGE_DELAY: Final = int( - os.environ.get("RESCAN_ON_FILESYSTEM_CHANGE_DELAY", 5) # 5 seconds + os.environ.get("RESCAN_ON_FILESYSTEM_CHANGE_DELAY", 5) # 5 minutes ) ENABLE_SCHEDULED_RESCAN: Final = ( os.environ.get("ENABLE_SCHEDULED_RESCAN", "false") == "true" diff --git a/backend/endpoints/scan.py b/backend/endpoints/scan.py index 886c47555..7609667c3 100644 --- a/backend/endpoints/scan.py +++ b/backend/endpoints/scan.py @@ -13,7 +13,9 @@ async def scan_platforms( - platform_slugs: list[str], complete_rescan: bool, selected_roms: list[str] + platform_slugs: list[str], + complete_rescan: bool = False, + selected_roms: list[str] = (), ): # Connect to external socketio server sm = ( diff --git a/backend/tasks/scan_library.py b/backend/tasks/scan_library.py index de382dcf1..71458cfe7 100644 --- a/backend/tasks/scan_library.py +++ b/backend/tasks/scan_library.py @@ -4,6 +4,8 @@ ENABLE_SCHEDULED_RESCAN, SCHEDULED_RESCAN_CRON, ) +from endpoints.scan import scan_platforms +from utils.redis import low_prio_queue from .exceptions import SchedulerException from . import tasks_scheduler @@ -17,16 +19,14 @@ def _get_existing_job(): return None -async def run(): +def run(): if not ENABLE_SCHEDULED_RESCAN: log.info("Scheduled library scan not enabled, unscheduling...") unschedule() return - from endpoints.scan import scan_platforms - log.info("Scheduled library scan started...") - await scan_platforms("", False) + low_prio_queue.enqueue(scan_platforms, platform_slugs=[]) log.info("Scheduled library scan done.") diff --git a/backend/watcher.py b/backend/watcher.py new file mode 100644 index 000000000..657428e0e --- /dev/null +++ b/backend/watcher.py @@ -0,0 +1,87 @@ +import os +from datetime import timedelta +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + +from endpoints.scan import scan_platforms +from utils.redis import low_prio_queue +from logger.logger import log + +from config import ( + HIGH_PRIO_STRUCTURE_PATH, + LIBRARY_BASE_PATH, + ENABLE_RESCAN_ON_FILESYSTEM_CHANGE, + RESCAN_ON_FILESYSTEM_CHANGE_DELAY, +) + +path = ( + HIGH_PRIO_STRUCTURE_PATH + if os.path.exists(HIGH_PRIO_STRUCTURE_PATH) + else LIBRARY_BASE_PATH +) + + +class EventHandler(FileSystemEventHandler): + def on_any_event(self, event): + if not ENABLE_RESCAN_ON_FILESYSTEM_CHANGE: + return + + # Ignore .DS_Store files + if event.src_path.endswith(".DS_Store"): + return + + # Ignore modified events + if event.event_type == "modified": + return + + event_src = event.src_path.split(path)[-1] + platform_slug = event_src.split("/")[1] + time_delta = timedelta(minutes=RESCAN_ON_FILESYSTEM_CHANGE_DELAY) + + log.info(f"Filesystem event: {event.event_type} {event_src}") + + low_prio_queue.scheduled_job_registry.remove_jobs() + + # Skip if a scan is already scheduled + for job_id in low_prio_queue.scheduled_job_registry.get_job_ids(): + job = low_prio_queue.fetch_job(job_id) + if ( + job + and job.is_scheduled + and job.func_name == "endpoints.scan.scan_platforms" + ): + if job.args[0] == []: + log.info("Full rescan already scheduled") + return + + if platform_slug in job.args[0]: + log.info(f"Scan already scheduled for {platform_slug}") + return + + rescan_in_msg = f"rescanning in {RESCAN_ON_FILESYSTEM_CHANGE_DELAY} minutes." + + # # Any change to a platform directory should trigger a full rescan + if event.is_directory and event_src.count("/") == 1: + log.info(f"Platform directory changed, {rescan_in_msg}") + return low_prio_queue.enqueue_in(time_delta, scan_platforms, []) + + # Otherwise trigger a rescan for the specific platform + log.info(f"Change detected in {platform_slug} folder, {rescan_in_msg}") + return low_prio_queue.enqueue_in( + time_delta, + scan_platforms, + [platform_slug], + ) + + +if __name__ == "__main__": + observer = Observer() + observer.schedule(EventHandler(), path, recursive=True) + observer.start() + + try: + while observer.is_alive(): + observer.join(1) + finally: + observer.stop() + observer.join() diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 4e57c2498..172eee0e9 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -12,6 +12,9 @@ # Start rq scheduler /init_scheduler & +# Start watchdog +/init_watchdog & + # Wait for any process to exit wait -n diff --git a/docker/init_scripts/init_scheduler b/docker/init_scripts/init_scheduler index c8dd46030..497cc97c4 100644 --- a/docker/init_scripts/init_scheduler +++ b/docker/init_scripts/init_scheduler @@ -1,4 +1,4 @@ #!/bin/bash cd /back -[[ ${ENABLE_EXPERIMENTAL_REDIS} == "true" ]] && rqscheduler --host ${REDIS_HOST} --port ${REDIS_PORT} || sleep infinity +[[ ${ENABLE_EXPERIMENTAL_REDIS} == "true" ]] && python3 scheduler.py || sleep infinity diff --git a/docker/init_scripts/init_watchdog b/docker/init_scripts/init_watchdog new file mode 100644 index 000000000..2d3b27f56 --- /dev/null +++ b/docker/init_scripts/init_watchdog @@ -0,0 +1,4 @@ +#!/bin/bash + +cd /back +python3 watcher.py diff --git a/docker/init_scripts/init_worker b/docker/init_scripts/init_worker index d1776b97c..f92c41664 100755 --- a/docker/init_scripts/init_worker +++ b/docker/init_scripts/init_worker @@ -1,4 +1,4 @@ #!/bin/bash cd /back -[[ ${ENABLE_EXPERIMENTAL_REDIS} == "true" ]] && rq worker high default low --url redis://${REDIS_HOST}:${REDIS_PORT}/0 || sleep infinity +[[ ${ENABLE_EXPERIMENTAL_REDIS} == "true" ]] && python3 worker.py || sleep infinity diff --git a/poetry.lock b/poetry.lock index c8df942b6..0aab17cab 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1967,6 +1967,45 @@ PyYAML = "*" wrapt = "*" yarl = "*" +[[package]] +name = "watchdog" +version = "3.0.0" +description = "Filesystem events monitoring" +optional = false +python-versions = ">=3.7" +files = [ + {file = "watchdog-3.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:336adfc6f5cc4e037d52db31194f7581ff744b67382eb6021c868322e32eef41"}, + {file = "watchdog-3.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a70a8dcde91be523c35b2bf96196edc5730edb347e374c7de7cd20c43ed95397"}, + {file = "watchdog-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:adfdeab2da79ea2f76f87eb42a3ab1966a5313e5a69a0213a3cc06ef692b0e96"}, + {file = "watchdog-3.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2b57a1e730af3156d13b7fdddfc23dea6487fceca29fc75c5a868beed29177ae"}, + {file = "watchdog-3.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7ade88d0d778b1b222adebcc0927428f883db07017618a5e684fd03b83342bd9"}, + {file = "watchdog-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7e447d172af52ad204d19982739aa2346245cc5ba6f579d16dac4bfec226d2e7"}, + {file = "watchdog-3.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9fac43a7466eb73e64a9940ac9ed6369baa39b3bf221ae23493a9ec4d0022674"}, + {file = "watchdog-3.0.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f"}, + {file = "watchdog-3.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc"}, + {file = "watchdog-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3"}, + {file = "watchdog-3.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3"}, + {file = "watchdog-3.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0"}, + {file = "watchdog-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8"}, + {file = "watchdog-3.0.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:13bbbb462ee42ec3c5723e1205be8ced776f05b100e4737518c67c8325cf6100"}, + {file = "watchdog-3.0.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346"}, + {file = "watchdog-3.0.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d"}, + {file = "watchdog-3.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33"}, + {file = "watchdog-3.0.0-py3-none-win32.whl", hash = "sha256:3ed7c71a9dccfe838c2f0b6314ed0d9b22e77d268c67e015450a29036a81f60f"}, + {file = "watchdog-3.0.0-py3-none-win_amd64.whl", hash = "sha256:4c9956d27be0bb08fc5f30d9d0179a855436e655f046d288e2bcc11adfae893c"}, + {file = "watchdog-3.0.0-py3-none-win_ia64.whl", hash = "sha256:5d9f3a10e02d7371cd929b5d8f11e87d4bad890212ed3901f9b4d68767bee759"}, + {file = "watchdog-3.0.0.tar.gz", hash = "sha256:4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9"}, +] + +[package.extras] +watchmedo = ["PyYAML (>=3.10)"] + [[package]] name = "wcwidth" version = "0.2.6" @@ -2231,4 +2270,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "3d06fb3f109d6d4c58998927f16c1b792e7624f22b987b4e138834303cc7df2b" +content-hash = "29a5ef6641ea9b51f2b0fc389713b0316bd9feb8b3d89db341bc71449ccead61" diff --git a/pyproject.toml b/pyproject.toml index 5b987e891..d045a9579 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ httpx = "^0.24.1" python-multipart = "^0.0.6" types-python-jose = "^3.3.4.8" types-passlib = "^1.7.7.13" +watchdog = "^3.0.0" [build-system] requires = ["poetry-core"]