Merge branch 'master' into scheduled-tasks

.github/workflows/build.yml

@@ -50,6 +50,9 @@ jobs:
             type=raw,value=${{ inputs.version }}
           labels: |
             org.opencontainers.image.version=${{ inputs.version }}
+            org.opencontainers.image.title="zurdi15/romm"
+            org.opencontainers.image.description="RomM (stands for Rom Manager) is a game library manager focused in retro gaming. Manage and organize all of your games from a web browser"
+            org.opencontainers.image.licenses="GPL-3.0"
       - name: Build image
         uses: docker/build-push-action@v4
         with:

CHANGELOG.md

@@ -1,3 +1,27 @@
+# v2.0.0 (_27-10-2023_)
+
+## Added
+ - User management system. Check the [docker-compose.example.yml](docker/docker-compose.example.yml) for all the needed changes and environment variables. Closes [#24](https://github.com/zurdi15/romm/issues/24)
+ - Gallery bulk selection. Closes [#50](https://github.com/zurdi15/romm/issues/50)
+ - Roms upload feature.
+ - Custom cover art.
+ - Custom name for `roms` folder throught the `ROMS_FOLDER_NAME` environment variable. Closes [#356](https://github.com/zurdi15/romm/issues/356)
+ - Added `IGDB_CLIENT_ID` and `IGDB_CLIENT_SECRET` as environment variables. `CLIENT_ID` and `CLIENT_ID` are deprecated and will be removed in future versions.
+ - Added icons for more platforms: CD-i, 3DO, Neo Geo Pocket Color, Nintendo 64DD, Satellaview, Playdia, Pippin, Mac
+
+## Fixed
+ - Fixed some checks before renaming a rom to avoid breaking names. Closes [#348](https://github.com/zurdi15/romm/issues/348)
+ - A lot of other minor bugs.
+
+## Changed
+ - RomM internal port changed from `80` to `8080.
+ - RomM docker image size reduced significantly.
+ - Improved scanning and IGDB requests returning first the exact match.
+ - Scan now times out at 4 hours to improve scans for larger libraries.
+ - Other minimal changes in platform icons.
+
+<br>
+
 # v1.10 (_15-08-2023_)
 
 ## Added
@@ -12,8 +36,6 @@
  - Improved scanning and IGDB requests logs. Fixes [#317](https://github.com/zurdi15/romm/issues/317)
  - Improved downloading process.  Fixes [#332](https://github.com/zurdi15/romm/issues/332)
 
-
-
 <!-- **_Note_**: Experimental support for redis in the backend was added for anyone that wants to test it (it's experimental so expect some bugs). It's not required yet (check docker-compose.example.yml to check how to set the needed environment variables) but it will likely be introduced in 2.0. -->
 
 <br>

README.md

@@ -26,6 +26,8 @@ Inspired by [Jellyfin](https://jellyfin.org/), allows you to manage all your gam
 * Scan your game library (all at once or by platform) and enriches it with IGDB metadata
 * Access your library via your web-browser
 * Possibility to select one of the matching IGDB results if the scan doesn't get the right one
+* Upload games directly from your web-browser
+* Set a custom cover for each game
 * EmuDeck folder structure compatibility
 * Multiple files games support
 * Download games directly from your web-browser
@@ -35,12 +37,6 @@ Inspired by [Jellyfin](https://jellyfin.org/), allows you to manage all your gam
 * Responsive design
 * Light and dark theme
 
-## 🛠 Roadmap
-
-* Upload games directly from your web-browser - [issue #54](https://github.com/zurdi15/romm/issues/54)
-* Manage save files directly from your web-browser - [issue #55](https://github.com/zurdi15/romm/issues/55)
-* Set a custom cover for each game - [issue #53](https://github.com/zurdi15/romm/issues/53)
-
 # Preview
 
 ## 🖥 Desktop
@@ -140,6 +136,15 @@ Review the [Platforms support](#platform-support) section for device naming conv
   │     ├─ rom_1.iso
   ```
 
+<h2 id="Authentication">🔒 Authentication</h2>
+
+If you want to enable the user management system, a redis container and some environment variables needs to be set. In the [docker-compose.yml](https://github.com/zurdi15/romm/blob/master/examples/docker-compose.example.yml) you will find the needed variables and an example of how to spin up a redis container:
+
+ - `ROMM_AUTH_ENABLED` and `ENABLE_EXPERIMENTAL_REDIS` must be set as `true`
+ - `ROMM_AUTH_SECRET_KEY` must be generated with `openssl rand -hex 32`
+ - `ROMM_AUTH_USERNAME` and `ROMM_AUTH_PASSWORD` can be set as wanted, being both `admin` by default.
+ - `REDIS_HOST` and `REDIS_PORT` must point to your redis instance
+
 <h2 id="configuration-file">⚙️ Configuration file</h2>
 
 RomM can be configured through a yml file.
@@ -300,5 +305,5 @@ Games can be tagged with region, revision or other tags using parenthesis in the
 
 # 🎖 Credits
 
-* PC icon support - <a href="https://www.flaticon.com/free-icons/keyboard-and-mouse" title="Keyboard and mouse icons">Keyboard and mouse icons created by Flat Icons - Flaticon</a>
+* Pc and Mac icon support - <a href="https://www.flaticon.com/free-icons/keyboard-and-mouse" title="Keyboard and mouse icons">Keyboard and mouse icons created by Flat Icons - Flaticon</a>
 * Default user icon - <a target="_blank" href="https://icons8.com/icon/tZuAOUGm9AuS/user-default">User Default</a> icon by <a target="_blank" href="https://icons8.com">Icons8</a>

backend/alembic/versions/1.9.2.py → backend/alembic/versions/2.0.0_.py

@@ -1,6 +1,6 @@
-"""empty message
+"""update to 2.0.0
 
-Revision ID: 583453a8a07f
+Revision ID: 2.0.0
 Revises: 1.8.3
 Create Date: 2023-08-10 22:18:24.012779
 
@@ -10,7 +10,7 @@
 
 
 # revision identifiers, used by Alembic.
-revision = "1.9.2"
+revision = "2.0.0"
 down_revision = "1.8.3"
 branch_labels = None
 depends_on = None

backend/config/__init__.py

@@ -11,13 +11,14 @@
 
 # PATHS
 ROMM_BASE_PATH: Final = os.environ.get("ROMM_BASE_PATH", "/romm")
+ROMS_FOLDER_NAME: Final = os.environ.get("ROMS_FOLDER_NAME", "roms")
 LIBRARY_BASE_PATH: Final = f"{ROMM_BASE_PATH}/library"
 FRONT_LIBRARY_PATH: Final = "/assets/romm/library"
 ROMM_USER_CONFIG_PATH: Final = f"{ROMM_BASE_PATH}/config.yml"
 SQLITE_DB_BASE_PATH: Final = f"{ROMM_BASE_PATH}/database"
 RESOURCES_BASE_PATH: Final = f"{ROMM_BASE_PATH}/resources"
 LOGS_BASE_PATH: Final = f"{ROMM_BASE_PATH}/logs"
-HIGH_PRIO_STRUCTURE_PATH: Final = f"{LIBRARY_BASE_PATH}/roms"
+HIGH_PRIO_STRUCTURE_PATH: Final = f"{LIBRARY_BASE_PATH}/{ROMS_FOLDER_NAME}"
 
 # DEFAULT RESOURCES
 DEFAULT_URL_COVER_L: Final = (

backend/endpoints/rom.py

@@ -200,14 +200,16 @@ async def update_rom(
     cleaned_data["file_name_no_tags"] = get_file_name_with_no_tags(
         cleaned_data["file_name"]
     )
+
     cleaned_data.update(
         fs.get_cover(
-            overwrite=True,
+            overwrite=not db_rom.has_cover,
             p_slug=db_platform.slug,
             r_name=cleaned_data["file_name_no_tags"],
             url_cover=cleaned_data.get("url_cover", ""),
         )
     )
+
     cleaned_data.update(
         fs.get_screenshots(
             p_slug=db_platform.slug,
@@ -221,21 +223,25 @@ async def update_rom(
         path_cover_l, path_cover_s, artwork_path = build_artwork_path(
             cleaned_data["r_name"], db_platform.fs_slug, file_ext
         )
+
         cleaned_data["path_cover_l"] = path_cover_l
         cleaned_data["path_cover_s"] = path_cover_s
-        file_location_l = f"{artwork_path}/big.{file_ext}"
-        file_location_s = f"{artwork_path}/small.{file_ext}"
+        cleaned_data["has_cover"] = 1
+
         artwork_file = artwork.file.read()
+        file_location_s = f"{artwork_path}/small.{file_ext}"
         with open(file_location_s, "wb+") as artwork_s:
             artwork_s.write(artwork_file)
+
+        file_location_l = f"{artwork_path}/big.{file_ext}"
         with open(file_location_l, "wb+") as artwork_l:
             artwork_l.write(artwork_file)
 
     dbh.update_rom(id, cleaned_data)
 
     return {
         "rom": dbh.get_rom(id),
-        "msg": f"Rom updated successfully!",
+        "msg": "Rom updated successfully!",
     }
 
 

backend/handler/igdb_handler.py

@@ -67,15 +67,19 @@ def _request(self, url: str, data: str, timeout: int = 120) -> list:
             res = requests.post(url, data, headers=self.headers, timeout=timeout)
             res.raise_for_status()
             return res.json()
-        except (HTTPError, Timeout) as err:
+        except HTTPError as err:
+            # Retry once if the auth token is invalid
             if err.response.status_code != 401:
                 log.error(err)
                 return []  # All requests to the IGDB API return a list
-
-        # Attempt to force a token refresh if the token is invalid
-        log.warning("Twitch token invalid: fetching a new one...")
-        token = self.twitch_auth._update_twitch_token()
-        self.headers["Authorization"] = f"Bearer {token}"
+
+            # Attempt to force a token refresh if the token is invalid
+            log.warning("Twitch token invalid: fetching a new one...")
+            token = self.twitch_auth._update_twitch_token()
+            self.headers["Authorization"] = f"Bearer {token}"
+        except Timeout:
+            # Retry once the request if it times out
+            pass
 
         try:
             res = requests.post(url, data, headers=self.headers, timeout=timeout)
@@ -215,7 +219,7 @@ async def get_rom(self, file_name: str, p_igdb_id: int):
             or self._search_rom(uc(search_term), p_igdb_id)
         )
 
-        r_igdb_id = res.get("id", 0)
+        r_igdb_id = res.get("id", "")
         r_slug = res.get("slug", "")
         r_name = res.get("name", search_term)
         summary = res.get("summary", "")

backend/handler/tests/test_igdb_handler.py

@@ -28,7 +28,7 @@ async def test_get_rom():
     assert urlparse(rom["url_screenshots"][0]).hostname == "images.igdb.com"
 
     rom = await igdbh.get_rom("Not a real game title", 4)
-    assert rom["r_igdb_id"] == 0
+    assert rom["r_igdb_id"] == ""
     assert rom["r_slug"] == ""
     assert rom["r_name"] == "Not a real game title"
     assert not rom["summary"]
@@ -39,7 +39,7 @@ async def test_get_rom():
 @pytest.mark.vcr()
 async def test_get_ps2_opl_rom():
     rom = await igdbh.get_rom("WWE Smack.iso", 8)
-    assert rom["r_igdb_id"] == 0
+    assert rom["r_igdb_id"] == ""
     assert rom["r_slug"] == ""
     assert rom["r_name"] == "WWE Smack"
     assert not rom["summary"]

backend/utils/__init__.py

@@ -58,14 +58,19 @@
 REGIONS_BY_SHORTCODE = {region[0].lower(): region[1] for region in REGIONS}
 REGIONS_NAME_KEYS = [region[1].lower() for region in REGIONS]
 
+TAG_REGEX = r"\(([^)]+)\)|\[([^]]+)\]"
+EXTENSION_REGEX = r"\.(\w+(\.\w+)*)$"
+
 
 def parse_tags(file_name: str) -> tuple:
     reg = ""
     rev = ""
     other_tags = []
-    tags = re.findall(r"\(([^)]+)", file_name)
+    tags = re.findall(TAG_REGEX, file_name)
+
+    for p_tag, s_tag in tags:
+        tag = p_tag or s_tag
 
-    for tag in tags:
         if tag.lower() in REGIONS_BY_SHORTCODE.keys():
             reg = REGIONS_BY_SHORTCODE[tag.lower()]
             continue
@@ -95,9 +100,13 @@ def parse_tags(file_name: str) -> tuple:
 
 
 def get_file_name_with_no_tags(file_name: str) -> str:
-    # Use .rsplit to remove only the file extension
-    return re.sub(r"[\(\[].*?[\)\]]", "", file_name.rsplit(".", 1)[0]).strip()
+    file_name_no_extension = re.sub(EXTENSION_REGEX, "", file_name).strip()
+    return re.sub(TAG_REGEX, "", file_name_no_extension).strip()
 
 
 def get_file_extension(rom: dict) -> str:
-    return rom["file_name"].split(".")[-1] if not rom["multi"] else ""
+    return (
+        re.search(EXTENSION_REGEX, rom["file_name"]).group(1)
+        if not rom["multi"]
+        else ""
+    )

backend/utils/fs.py

@@ -3,10 +3,12 @@
 from pathlib import Path
 import datetime
 import requests
+from urllib.parse import quote
 
 from config import (
     LIBRARY_BASE_PATH,
     HIGH_PRIO_STRUCTURE_PATH,
+    ROMS_FOLDER_NAME,
     RESOURCES_BASE_PATH,
     DEFAULT_URL_COVER_L,
     DEFAULT_PATH_COVER_L,
@@ -63,19 +65,21 @@ def _get_cover_path(p_slug: str, r_name: str, size: str):
 
     Args:
         p_slug: short name of the platform
-        file_name: name of rom file
+        r_name: name of rom
         size: size of the cover -> big | small
     """
     strtime = str(datetime.datetime.now().timestamp())
     return f"{p_slug}/{r_name}/cover/{size}.png?timestamp={strtime}"
 
 
 def get_cover(overwrite: bool, p_slug: str, r_name: str, url_cover: str = "") -> dict:
+    rom_name = quote(r_name)
+
     # Cover small
     if (overwrite or not _cover_exists(p_slug, r_name, "small")) and url_cover:
         _store_cover(p_slug, r_name, url_cover, "small")
     path_cover_s = (
-        _get_cover_path(p_slug, r_name, "small")
+        _get_cover_path(p_slug, rom_name, "small")
         if _cover_exists(p_slug, r_name, "small")
         else DEFAULT_PATH_COVER_S
     )
@@ -84,7 +88,7 @@ def get_cover(overwrite: bool, p_slug: str, r_name: str, url_cover: str = "") ->
     if (overwrite or not _cover_exists(p_slug, r_name, "big")) and url_cover:
         _store_cover(p_slug, r_name, url_cover, "big")
     (path_cover_l, has_cover) = (
-        (_get_cover_path(p_slug, r_name, "big"), 1)
+        (_get_cover_path(p_slug, rom_name, "big"), 1)
         if _cover_exists(p_slug, r_name, "big")
         else (DEFAULT_PATH_COVER_L, 0)
     )
@@ -101,7 +105,7 @@ def _store_screenshot(p_slug: str, r_name: str, url: str, idx: int):
 
     Args:
         p_slug: short name of the platform
-        file_name: name of rom file
+        r_name: name of rom
         url: url to get the screenshot
     """
     screenshot_file: str = f"{idx}.jpg"
@@ -118,17 +122,19 @@ def _get_screenshot_path(p_slug: str, r_name: str, idx: str):
 
     Args:
         p_slug: short name of the platform
-        file_name: name of rom file
+        r_name: name of rom
         idx: index number of screenshot
     """
     return f"{p_slug}/{r_name}/screenshots/{idx}.jpg"
 
 
 def get_screenshots(p_slug: str, r_name: str, url_screenshots: list) -> dict:
+    rom_name = quote(r_name)
+
     path_screenshots: list[str] = []
     for idx, url in enumerate(url_screenshots):
         _store_screenshot(p_slug, r_name, url, idx)
-        path_screenshots.append(_get_screenshot_path(p_slug, r_name, str(idx)))
+        path_screenshots.append(_get_screenshot_path(p_slug, rom_name, str(idx)))
     return {"path_screenshots": path_screenshots}
 
 
@@ -172,9 +178,9 @@ def get_platforms() -> list[str]:
 # ========= Roms utils =========
 def get_roms_structure(p_slug: str):
     return (
-        f"roms/{p_slug}"
+        f"{ROMS_FOLDER_NAME}/{p_slug}"
         if os.path.exists(HIGH_PRIO_STRUCTURE_PATH)
-        else f"{p_slug}/roms"
+        else f"{p_slug}/{ROMS_FOLDER_NAME}"
     )
 
 
@@ -308,9 +314,11 @@ def build_upload_roms_path(p_slug: str):
 
 
 def build_artwork_path(r_name: str, p_slug: str, file_ext: str):
+    rom_name = quote(r_name)
     strtime = str(datetime.datetime.now().timestamp())
-    path_cover_l = f"{p_slug}/{r_name}/cover/big.{file_ext}?timestamp={strtime}"
-    path_cover_s = f"{p_slug}/{r_name}/cover/small.{file_ext}?timestamp={strtime}"
+
+    path_cover_l = f"{p_slug}/{rom_name}/cover/big.{file_ext}?timestamp={strtime}"
+    path_cover_s = f"{p_slug}/{rom_name}/cover/small.{file_ext}?timestamp={strtime}"
     artwork_path = f"{RESOURCES_BASE_PATH}/{p_slug}/{r_name}/cover"
     Path(artwork_path).mkdir(parents=True, exist_ok=True)
     return path_cover_l, path_cover_s, artwork_path