Add mame xml and scheduled tasks

.gitignore

@@ -55,3 +55,4 @@ frontend/dev-dist
 
 # outside data
 switch_titledb.json
+mame.xml

backend/config/__init__.py

@@ -82,5 +82,11 @@
     os.environ.get("ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB", "false") == "true"
 )
 SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: Final = os.environ.get(
-    "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 3 * * *"  # At 3:00 AM every day
+    "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON", "0 4 * * *"  # At 4:00 AM every day
+)
+ENABLE_SCHEDULED_UPDATE_MAME_XML: Final = (
+    os.environ.get("ENABLE_SCHEDULED_UPDATE_MAME_XML", "false") == "true"
+)
+SCHEDULED_UPDATE_MAME_XML_CRON: Final = os.environ.get(
+    "SCHEDULED_UPDATE_MAME_XML_CRON", "0 4 * * *"  # At 4:00 AM every day
 )

backend/endpoints/tests/test_heartbeat.py

@@ -13,7 +13,9 @@ def test_heartbeat():
         'ENABLE_RESCAN_ON_FILESYSTEM_CHANGE': True,
         'ENABLE_SCHEDULED_RESCAN': True,
         'ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB': True,
+        'ENABLE_SCHEDULED_UPDATE_MAME_XML': True,
         'RESCAN_ON_FILESYSTEM_CHANGE_DELAY': 5,
         'SCHEDULED_RESCAN_CRON': '0 3 * * *',
-        'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 3 * * *',
+        'SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON': '0 4 * * *',
+        'SCHEDULED_UPDATE_MAME_XML_CRON': '0 4 * * *',
     }

backend/handler/igdb_handler.py

@@ -6,6 +6,7 @@
 import time
 import os
 import json
+import xmltodict
 from unidecode import unidecode as uc
 from requests.exceptions import HTTPError, Timeout
 from typing import Final
@@ -15,12 +16,14 @@
 from logger.logger import log
 from utils.cache import cache
 from tasks.update_switch_titledb import update_switch_titledb_task
+from tasks.update_mame_xml import update_mame_xml_task
 
 MAIN_GAME_CATEGORY: Final = 0
 EXPANDED_GAME_CATEGORY: Final = 10
 N_SCREENSHOTS: Final = 5
 PS2_IGDB_ID: Final = 8
 SWITCH_IGDB_ID: Final = 130
+ARCADE_IGDB_ID: Final = 52
 
 PS2_OPL_REGEX: Final = r"^([A-Z]{4}_\d{3}\.\d{2})\..*$"
 PS2_OPL_INDEX_FILE: Final = os.path.join(
@@ -32,6 +35,8 @@
     os.path.dirname(__file__), "fixtures", "switch_titledb.json"
 )
 
+MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml")
+
 
 class IGDBHandler:
     def __init__(self) -> None:
@@ -183,6 +188,27 @@ async def get_rom(self, file_name: str, p_igdb_id: int):
                 if index_entry:
                     search_term = index_entry["name"]  # type: ignore
 
+        if p_igdb_id == ARCADE_IGDB_ID:
+            mame_index = {}
+
+            try:
+                with open(MAME_XML_FILE, "r") as index_xml:
+                    mame_index = xmltodict.parse(index_xml.read())
+            except FileNotFoundError:
+                log.warning("Fetching the MAME XML file from HyperspinFE...")
+                await update_mame_xml_task.run(force=True)
+
+                with open(MAME_XML_FILE, "r") as index_xml:
+                    mame_index = xmltodict.parse(index_xml.read())
+            finally:
+                index_entry = [
+                    game
+                    for game in mame_index["menu"]["game"]
+                    if game["@name"] == search_term
+                ]
+                if index_entry:
+                    search_term = index_entry[0].get("description", search_term)
+
         res = (
             self._search_rom(uc(search_term), p_igdb_id, MAIN_GAME_CATEGORY)
             or self._search_rom(uc(search_term), p_igdb_id, EXPANDED_GAME_CATEGORY)

backend/main.py

@@ -19,6 +19,8 @@
     SCHEDULED_RESCAN_CRON,
     ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
     SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+    ENABLE_SCHEDULED_UPDATE_MAME_XML,
+    SCHEDULED_UPDATE_MAME_XML_CRON,
 )
 from endpoints import search, platform, rom, identity, oauth, scan  # noqa
 from handler import dbh
@@ -82,6 +84,8 @@ def heartbeat():
         "SCHEDULED_RESCAN_CRON": SCHEDULED_RESCAN_CRON,
         "ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB": ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,  # noqa
         "SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON": SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+        "ENABLE_SCHEDULED_UPDATE_MAME_XML": ENABLE_SCHEDULED_UPDATE_MAME_XML,
+        "SCHEDULED_UPDATE_MAME_XML_CRON": SCHEDULED_UPDATE_MAME_XML_CRON,
     }
 
 

backend/scheduler.py

@@ -5,6 +5,7 @@
 from logger.logger import log
 from tasks.scan_library import scan_library_task
 from tasks.update_switch_titledb import update_switch_titledb_task
+from tasks.update_mame_xml import update_mame_xml_task
 
 if __name__ == "__main__":
     if not ENABLE_EXPERIMENTAL_REDIS:
@@ -13,6 +14,7 @@
     # Initialize the tasks
     scan_library_task.init()
     update_switch_titledb_task.init()
+    update_mame_xml_task.init()
 
     log.info("Starting scheduler")
 

backend/tasks/update_mame_xml.py

@@ -0,0 +1,28 @@
+import os
+from pathlib import Path
+
+from typing import Final
+from config import (
+    ENABLE_SCHEDULED_UPDATE_MAME_XML,
+    SCHEDULED_UPDATE_MAME_XML_CRON,
+)
+from .utils import RemoteFilePullTask
+
+FIXTURE_FILE_PATH: Final = (
+    Path(os.path.dirname(__file__)).parent / "handler" / "fixtures" / "mame.xml"
+)
+
+
+class UpdateMAMEXMLTask(RemoteFilePullTask):
+    def __init__(self):
+        super().__init__(
+            func="tasks.update_mame_xml.update_mame_xml_task.run",
+            description="mame xml update",
+            enabled=ENABLE_SCHEDULED_UPDATE_MAME_XML,
+            cron_string=SCHEDULED_UPDATE_MAME_XML_CRON,
+            url="https://hyperlist.hyperspin-fe.com/genall.php?system=6",
+            file_path=FIXTURE_FILE_PATH,
+        )
+
+
+update_mame_xml_task = UpdateMAMEXMLTask()

backend/tasks/update_switch_titledb.py

@@ -1,52 +1,31 @@
-import requests
 import os
 from pathlib import Path
 
-from logger.logger import log
 from typing import Final
 from config import (
     ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
     SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
 )
-from .utils import PeriodicTask
+from .utils import RemoteFilePullTask
 
-RAW_URL: Final = "https://raw.githubusercontent.com/blawar/titledb/master/US.en.json"
-FIXTURE_FILE_PATH = (
+FIXTURE_FILE_PATH: Final = (
     Path(os.path.dirname(__file__)).parent
     / "handler"
     / "fixtures"
     / "switch_titledb.json"
 )
 
 
-class UpdateSwitchTitleDBTask(PeriodicTask):
+class UpdateSwitchTitleDBTask(RemoteFilePullTask):
     def __init__(self):
         super().__init__(
             func="tasks.update_switch_titledb.update_switch_titledb_task.run",
             description="switch titledb update",
             enabled=ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB,
             cron_string=SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON,
+            url="https://raw.githubusercontent.com/blawar/titledb/master/US.en.json",
+            file_path=FIXTURE_FILE_PATH,
         )
 
-    async def run(self, force: bool = False):
-        if not ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB and not force:
-            log.info("Scheduled switch titledb update not enabled, unscheduling...")
-            self.unschedule()
-            return
-
-        log.info("Scheduled switch titledb update started...")
-
-        try:
-            response = requests.get(RAW_URL)
-            response.raise_for_status()
-
-            with open(FIXTURE_FILE_PATH, "wb") as fixture:
-                fixture.write(response.content)
-
-            log.info("Scheduled switch titledb update done")
-        except requests.exceptions.RequestException as e:
-            log.error("Scheduled switch titledb update failed", exc_info=True)
-            log.error(e)
-
 
 update_switch_titledb_task = UpdateSwitchTitleDBTask()

backend/tasks/utils.py

@@ -1,3 +1,4 @@
+import requests
 from rq_scheduler import Scheduler
 from abc import ABC, abstractmethod
 
@@ -73,3 +74,31 @@ def unschedule(self):
 
         tasks_scheduler.cancel(job)
         log.info(f"{self.description.capitalize()} unscheduled.")
+
+
+class RemoteFilePullTask(PeriodicTask):
+    def __init__(self, *args, url: str, file_path: str, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.url = url
+        self.file_path = file_path
+
+    async def run(self, force: bool = False):
+        if not self.enabled and not force:
+            log.info(f"Scheduled {self.description} not enabled, unscheduling...")
+            self.unschedule()
+            return
+
+        log.info(f"Scheduled {self.description} started...")
+
+        try:
+            response = requests.get(self.url)
+            response.raise_for_status()
+
+            with open(self.file_path, "wb") as fixture:
+                fixture.write(response.content)
+
+            log.info(f"Scheduled {self.description} done")
+        except requests.exceptions.RequestException as e:
+            log.error(f"Scheduled {self.description} failed", exc_info=True)
+            log.error(e)

env.template

@@ -39,3 +39,5 @@ ENABLE_SCHEDULED_RESCAN=true
 SCHEDULED_RESCAN_CRON=0 3 * * *
 ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true
 SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 4 * * *
+ENABLE_SCHEDULED_UPDATE_MAME_XML=true
+SCHEDULED_UPDATE_MAME_XML_CRON=0 4 * * *

frontend/src/views/Settings/General/Settings.vue

@@ -46,10 +46,17 @@ onBeforeMount(async () => {
   switchUpdate =
     switchUpdate.charAt(0).toLocaleLowerCase() + switchUpdate.substr(1);
 
+  let mameUpdate = cronstrue.toString(
+    data.SCHEDULED_UPDATE_MAME_XML_CRON,
+    { verbose: true }
+  );
+  mameUpdate = mameUpdate.charAt(0).toLocaleLowerCase() + mameUpdate.substr(1);
+
   heartbeat.value = {
     ...data,
     SCHEDULED_RESCAN_CRON: rescan,
     SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON: switchUpdate,
+    SCHEDULED_UPDATE_MAME_XML_CRON: mameUpdate,
   };
 });
 </script>
@@ -175,6 +182,32 @@ onBeforeMount(async () => {
             </p>
           </div>
         </v-col>
+        <v-col
+          cols="12"
+          md="4"
+          sm="6"
+          :class="{
+            'status-item d-flex': true,
+            disabled: !heartbeat.ENABLE_SCHEDULED_UPDATE_MAME_XML,
+          }"
+        >
+          <v-icon
+            :icon="
+              heartbeat.ENABLE_SCHEDULED_UPDATE_MAME_XML
+                ? 'mdi-clock-check-outline'
+                : 'mdi-clock-remove-outline'
+            "
+          />
+          <div class="ml-3">
+            <v-label class="font-weight-bold">
+              Scheduled MAME XML update
+            </v-label>
+            <p class="mt-1">
+              Updates the Nintedo MAME XML file
+              {{ heartbeat.SCHEDULED_UPDATE_MAME_XML_CRON }}
+            </p>
+          </div>
+        </v-col>
       </v-row>
     </v-card-text>
   </v-card>

pyproject.toml

@@ -45,6 +45,7 @@ python-multipart = "^0.0.6"
 types-python-jose = "^3.3.4.8"
 types-passlib = "^1.7.7.13"
 watchdog = "^3.0.0"
+xmltodict = "^0.13.0"
 
 [build-system]
 requires = ["poetry-core"]

pytest.ini

@@ -13,6 +13,8 @@ env =
    ENABLE_RESCAN_ON_FILESYSTEM_CHANGE=true
    ENABLE_SCHEDULED_RESCAN=true
    ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB=true
+   ENABLE_SCHEDULED_UPDATE_MAME_XML=true
    RESCAN_ON_FILESYSTEM_CHANGE_DELAY=5
    SCHEDULED_RESCAN_CRON=0 3 * * *
    SCHEDULED_UPDATE_SWITCH_TITLEDB_CRON=0 3 * * *
+   SCHEDULED_UPDATE_MAME_XML_CRON=0 3 * * *