Add scheduled tasks for library scan + titledb update

.gitignore

@@ -52,3 +52,6 @@ backend/romm_test/logs
 
 # service worker
 frontend/dev-dist
+
+# outside data
+switch_titledb.json

backend/endpoints/scan.py

@@ -1,15 +1,12 @@
 import emoji
 import socketio  # type: ignore
-from rq import Queue
 
 from logger.logger import log
 from utils import fs, fastapi
 from utils.exceptions import PlatformsNotFoundException, RomsNotFoundException
 from handler import dbh
 from utils.socket import socket_server
-from utils.cache import redis_client, redis_url, redis_connectable
-
-scan_queue = Queue(connection=redis_client)
+from utils.redis import high_prio_queue, redis_url, redis_connectable
 
 
 async def scan_platforms(paltforms: str, complete_rescan: bool):
@@ -83,6 +80,6 @@ async def scan_handler(_sid: str, platforms: str, complete_rescan: bool = True):
 
     # Run in worker if redis is available
     if redis_connectable:
-        return scan_queue.enqueue(scan_platforms, platforms, complete_rescan)
+        return high_prio_queue.enqueue(scan_platforms, platforms, complete_rescan)
     else:
         await scan_platforms(platforms, complete_rescan)

backend/handler/ps2_opl_index.py → backend/handler/fixtures/ps2_opl_index.json

@@ -1,4 +1,4 @@
-opl_index = {
+{
   "SLES_556.71": {
     "Name": "Fifa '14",
     "Region": "PAL"

backend/handler/igdb_handler.py

@@ -4,23 +4,31 @@
 import requests
 import re
 import time
+import os
+import json
 from unidecode import unidecode as uc
 from requests.exceptions import HTTPError, Timeout
-from typing import Optional
 
 from config import CLIENT_ID, CLIENT_SECRET
 from utils import get_file_name_with_no_tags as get_search_term
 from logger.logger import log
 from utils.cache import cache
-from .ps2_opl_index import opl_index
 
 MAIN_GAME_CATEGORY = 0
 EXPANDED_GAME_CATEGORY = 10
-
 N_SCREENSHOTS = 5
+PS2_IGDB_ID = 8
+SWITCH_IGDB_ID = 130
 
 ps2_opl_regex = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$"
-PS2_IGDB_ID = 8
+ps2_opl_index_file = os.path.join(
+    os.path.dirname(__file__), "fixtures", "ps2_opl_index.json"
+)
+
+switch_titledb_regex = r"^(70[0-9]{12})$"
+switch_titledb_index_file = os.path.join(
+    os.path.dirname(__file__), "fixtures", "switch_titledb.json"
+)
 
 
 class IGDBHandler:
@@ -125,9 +133,23 @@ def get_rom(self, file_name: str, p_igdb_id: int):
         match = re.match(ps2_opl_regex, search_term)
         if p_igdb_id == PS2_IGDB_ID and match:
             serial_code = match.group(1)
-            index_entry = opl_index.get(serial_code, None)
-            if index_entry:
-                search_term = index_entry["Name"]  # type: ignore
+
+            with open(ps2_opl_index_file, "r") as index_json:
+                opl_index = json.loads(index_json.read())
+                index_entry = opl_index.get(serial_code, None)
+                if index_entry:
+                    search_term = index_entry["Name"]  # type: ignore
+
+        # Patch support for switch titleID filename format
+        match = re.match(switch_titledb_regex, search_term)
+        if p_igdb_id == SWITCH_IGDB_ID and match:
+            title_id = match.group(1)
+
+            with open(switch_titledb_index_file, "r") as index_json:
+                titledb_index = json.loads(index_json.read())
+                index_entry = titledb_index.get(title_id, None)
+                if index_entry:
+                    search_term = index_entry["name"]  # type: ignore
 
         res = (
             self._search_rom(uc(search_term), p_igdb_id, MAIN_GAME_CATEGORY)

backend/tasks/__init__.py

@@ -0,0 +1,5 @@
+from rq_scheduler import Scheduler
+
+from utils.redis import low_prio_queue
+
+scheduler = Scheduler(queue=low_prio_queue, connection=low_prio_queue.connection)

backend/tasks/scan_library.py

@@ -0,0 +1,36 @@
+from utils.redis import redis_connectable
+from logger.logger import log
+from . import scheduler
+from utils.exceptions import SchedulerException
+
+
+async def run():
+    from endpoints.scan import scan_platforms
+
+    log.info("Scheduled library scan started...")
+    await scan_platforms("", False)
+    log.info("Scheduled library scan done.")
+
+
+def schedule():
+    if not redis_connectable:
+        raise SchedulerException("Redis not connectable, library scan not scheduled.")
+
+    existing_jobs = scheduler.get_jobs(func_name="tasks.scan_library.run")
+    if existing_jobs:
+        raise SchedulerException("Library scan already scheduled.")
+
+    return scheduler.cron(
+        "0 3 * * *",  # At 3:00 AM every day
+        func="tasks.scan_library.run",
+        repeat=None,
+    )
+
+
+def unschedule():
+    existing_jobs = scheduler.get_jobs(func_name="tasks.scan_library.run")
+
+    if not existing_jobs:
+        raise SchedulerException("No library scan scheduled.")
+
+    scheduler.cancel(*existing_jobs)

backend/tasks/update_switch_titledb.py

@@ -0,0 +1,59 @@
+import requests
+import os
+from pathlib import Path
+
+from utils.redis import redis_connectable
+from logger.logger import log
+from typing import Final
+from . import scheduler
+from utils.exceptions import SchedulerException
+
+RAW_URL: Final = "https://raw.githubusercontent.com/blawar/titledb/master/US.en.json"
+FIXTURE_FILE_PATH = (
+    Path(os.path.dirname(__file__)).parent
+    / "handler"
+    / "fixtures"
+    / "switch_titledb.json"
+)
+
+print(FIXTURE_FILE_PATH)
+
+
+async def run():
+    log.info("Scheduled TitleDB update started...")
+
+    try:
+        response = requests.get(RAW_URL)
+        response.raise_for_status()
+
+        with open(FIXTURE_FILE_PATH, "wb") as fixture:
+            fixture.write(response.content)
+
+        log.info("TitleDB update done.")
+    except requests.exceptions.RequestException as e:
+        log.error("TitleDB update failed.", exc_info=True)
+        log.error(e)
+
+
+def schedule():
+    if not redis_connectable:
+        raise SchedulerException("Redis not connectable, titleDB update not scheduled.")
+
+    existing_jobs = scheduler.get_jobs(func_name="tasks.update_switch_titledb.run")
+    if existing_jobs:
+        raise SchedulerException("TitleDB update already scheduled.")
+
+    return scheduler.cron(
+        "0 3 * * *",  # At 3:00 AM every day
+        func="tasks.update_switch_titledb.run",
+        repeat=None,
+    )
+
+
+def unschedule():
+    existing_jobs = scheduler.get_jobs(func_name="tasks.update_switch_titledb.run")
+
+    if not existing_jobs:
+        raise SchedulerException("No TitleDB update scheduled.")
+
+    scheduler.cancel(*existing_jobs)

backend/utils/cache.py

@@ -1,15 +1,7 @@
-from redis import Redis, ConnectionError
+from redis import Redis
 
 from config import REDIS_HOST, REDIS_PORT
-
-redis_client = Redis(host=REDIS_HOST, port=int(REDIS_PORT), db=0)
-redis_url = f"redis://{REDIS_HOST}:{REDIS_PORT}"
-
-try:
-    redis_connectable = redis_client.ping()
-except ConnectionError:
-    redis_connectable = False
-
+from .redis import redis_connectable
 
 class FallbackCache:
     def __init__(self) -> None:

backend/utils/exceptions.py

@@ -35,3 +35,12 @@ def __init__(self, rom_name: str):
 
     def __repr__(self):
         return self.message
+
+
+class SchedulerException(Exception):
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(self.message)
+
+    def __repr__(self):
+        return self.message

backend/utils/redis.py

@@ -0,0 +1,17 @@
+from redis import Redis, ConnectionError
+from rq import Queue
+
+from config import REDIS_HOST, REDIS_PORT
+
+
+redis_client = Redis(host=REDIS_HOST, port=int(REDIS_PORT), db=0)
+redis_url = f"redis://{REDIS_HOST}:{REDIS_PORT}"
+
+try:
+    redis_connectable = redis_client.ping()
+except ConnectionError:
+    redis_connectable = False
+
+high_prio_queue = Queue(name="high", connection=redis_client)
+default_queue = Queue(name="default", connection=redis_client)
+low_prio_queue = Queue(name="low", connection=redis_client)

backend/utils/socket.py

@@ -1,6 +1,6 @@
 import socketio  # type: ignore
 
-from utils.cache import redis_url, redis_connectable
+from utils.redis import redis_url, redis_connectable
 
 
 socket_server = socketio.AsyncServer(

backend/worker.py

@@ -1,7 +1,7 @@
 import sys
 from rq import Worker, Queue, Connection
 
-from utils.cache import redis_client, redis_connectable
+from utils.redis import redis_client, redis_connectable
 
 listen = ["high", "default", "low"]
 

pyproject.toml

@@ -34,6 +34,7 @@ types-requests = "^2.31.0.2"
 mypy = "^1.4.1"
 types-redis = "^4.6.0.3"
 stream-zip = "^0.0.67"
+rq-scheduler = "^0.13.1"
 
 [build-system]
 requires = ["poetry-core"]