Importer fixes (d4lfteam#271)

src/__init__.py

@@ -1 +1 @@
-__version__ = "5.5.1"
+__version__ = "5.5.2"

src/gui/importer/common.py

@@ -79,6 +79,22 @@ def format_number_as_short_string(n: int) -> str:
     return f"{int(result)}M" if result.is_integer() else f"{result:.2f}M"
 
 
+def get_class_name(input_str: str) -> str:
+    input_str = input_str.lower()
+    if "barbarian" in input_str:
+        return "Barbarian"
+    if "druid" in input_str:
+        return "Druid"
+    if "necromancer" in input_str:
+        return "Necromancer"
+    if "rogue" in input_str:
+        return "Rogue"
+    if "sorcerer" in input_str:
+        return "Sorcerer"
+    Logger.error(f"Couldn't match class name {input_str=}")
+    return "Unknown"
+
+
 def get_with_retry(url: str) -> requests.Response:
     for _ in range(5):
         r = requests.get(url)

src/gui/importer/d4builds.py

@@ -15,6 +15,7 @@
 from src.gui.importer.common import (
     fix_offhand_type,
     fix_weapon_type,
+    get_class_name,
     match_to_enum,
     retry_importer,
     save_as_profile,
@@ -26,7 +27,7 @@
 
 BASE_URL = "https://d4builds.gg/builds"
 BUILD_OVERVIEW_XPATH = "//*[@class='builder__stats__list']"
-CLASS_XPATH = ".//*[contains(@class, 'builder__header__description')]/*"
+CLASS_XPATH = "//*[contains(@class, 'builder__header__description')]"
 ITEM_GROUP_XPATH = ".//*[contains(@class, 'builder__stats__group')]"
 ITEM_SLOT_XPATH = ".//*[contains(@class, 'builder__stats__slot')]"
 ITEM_STATS_XPATH = ".//*[contains(@class, 'dropdown__button__wrapper')]"
@@ -54,7 +55,10 @@ def import_d4builds(driver: ChromiumDriver = None, url: str = None):
     wait.until(EC.presence_of_element_located((By.XPATH, PAPERDOLL_XPATH)))
     time.sleep(5)  # super hacky but I didn't find anything else. The page is not fully loaded when the above wait is done
     data = lxml.html.fromstring(driver.page_source)
-    class_name = data.xpath(CLASS_XPATH)[0].tail.lower()
+    if (elem := data.xpath(CLASS_XPATH + "/*")) or (elem := data.xpath(CLASS_XPATH)):  # noqa SIM114
+        class_name = get_class_name(f"{elem[0].tail} {elem[0].text}")
+    else:
+        class_name = "Unknown"
     if not (items := data.xpath(BUILD_OVERVIEW_XPATH)):
         Logger.error(msg := "No items found")
         raise D4BuildsException(msg)
@@ -160,6 +164,7 @@ def _get_non_unique_slots(data: lxml.html.HtmlElement) -> list[str]:
     os.chdir(pathlib.Path(__file__).parent.parent.parent.parent)
     URLS = [
         "https://d4builds.gg/builds/463e7337-8fa9-491f-99a0-cbd6c65fc6f4/?var=1",
+        "https://d4builds.gg/builds/b5d603bb-4442-42e8-a84d-962e6e42344c?var=0",
     ]
     for X in URLS:
         import_d4builds(url=X)

src/gui/importer/diablo_trade.py

@@ -1,10 +1,13 @@
 import dataclasses
 import datetime
+import json
 import os
 import pathlib
 from typing import Any
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
+from pydantic import ValidationError
+
 from src.config.models import AffixFilterCountModel, AffixFilterModel, ItemFilterModel, ProfileModel
 from src.dataloader import Dataloader
 from src.gui.importer.common import (
@@ -38,6 +41,11 @@ class _Listing:
     item_rarity: ItemRarity | None = None
     item_type: ItemType | None = None
     price: int = 0
+    raw_data: dict[str, Any] | None = None
+
+
+class DiabloTradeException(Exception):
+    pass
 
 
 @retry_importer
@@ -67,6 +75,7 @@ def import_diablo_trade(url: str, max_listings: int):
                 item_rarity=match_to_enum(enum_class=ItemRarity, target_string=listing["rarity"]),
                 item_type=match_to_enum(enum_class=ItemType, target_string=listing["itemType"]),
                 price=listing["price"],
+                raw_data=listing,
             )
             try:
                 assert listing_obj.item_type is not None
@@ -79,7 +88,15 @@ def import_diablo_trade(url: str, max_listings: int):
         if len(all_listings) >= max_listings:
             break
         cursor += 1
-    profile = ProfileModel(name="diablo_trade", Affixes=_create_filters_from_items(items=all_listings))
+
+    try:
+        profile = ProfileModel(name="diablo_trade", Affixes=_create_filters_from_items(items=all_listings))
+    except ValidationError as exc:
+        Logger.exception(msg := "Failed to validate profile. Dumping data for debugging.")
+        with open(f"diablo_trade_dump_{datetime.datetime.now(tz=datetime.UTC).strftime("%Y-%m-%d %H:%M:%S")}.json", "w") as f:
+            json.dump(all_listings, f, indent=4, sort_keys=True)
+        raise DiabloTradeException(msg) from exc
+
     Logger.info(f"Saving profile with {len(profile.Affixes)} filters")
     save_as_profile(
         file_name=f"diablo_trade_{datetime.datetime.now(tz=datetime.UTC).strftime("%Y_%m_%d_%H_%M_%S")}", profile=profile, url=url
@@ -154,7 +171,8 @@ def _create_filters_from_items(items: list[_Listing]) -> list[dict[str, ItemFilt
     Logger.init("debug")
     os.chdir(pathlib.Path(__file__).parent.parent.parent.parent)
     URLS = [
-        "https://diablo.trade/listings/items?exactPrice=true&rarity=legendary&sold=true&sort=newest",
+        # "https://diablo.trade/listings/items?exactPrice=true&rarity=legendary&sold=true&sort=newest",
+        "https://diablo.trade/listings/items?categories=ancestral&equipment=amulet,boots,chestarmor,gloves,helm,pants,ring,axe,bow,crossbow,dagger,mace,scythe,staff,sword,twohandedaxe,twohandedmace,twohandedscythe,twohandedsword,wand,focus,shield,totem&exactPrice=true&itemType=equipment&level=80,100&mode=season%20softcore&power=925,1000&price=1000000,9999999999&sold=true&sort=newest&cursor=1"
     ]
     for x in URLS:
         import_diablo_trade(url=x, max_listings=200)

src/gui/importer/maxroll.py

@@ -166,19 +166,19 @@ def _extract_planner_url_and_id_from_planner(url: str) -> tuple[str, int]:
 
         try:
             r = get_with_retry(url=PLANNER_API_BASE_URL + planner_id)
-        except ConnectionError as ex:
-            Logger.error(msg := "Couldn't get planner")
-            raise MaxrollException(msg) from ex
+        except ConnectionError as exc:
+            Logger.exception(msg := "Couldn't get planner")
+            raise MaxrollException(msg) from exc
         data_id = json.loads(r.json()["data"])["activeProfile"]
     return PLANNER_API_BASE_URL + planner_id, data_id
 
 
 def _extract_planner_url_and_id_from_guide(url: str) -> tuple[str, int]:
     try:
         r = get_with_retry(url=url)
-    except ConnectionError as ex:
-        Logger.error(msg := "Couldn't get build guide")
-        raise MaxrollException(msg) from ex
+    except ConnectionError as exc:
+        Logger.exception(msg := "Couldn't get build guide")
+        raise MaxrollException(msg) from exc
     data = lxml.html.fromstring(r.text)
     if not (embed := data.xpath(BUILD_GUIDE_PLANNER_EMBED_XPATH)):
         Logger.error(msg := "Couldn't find planner url in build guide")

src/gui/importer/mobalytics.py

@@ -7,7 +7,15 @@
 
 from src.config.models import AffixFilterCountModel, AffixFilterModel, ItemFilterModel, ProfileModel
 from src.dataloader import Dataloader
-from src.gui.importer.common import fix_offhand_type, fix_weapon_type, get_with_retry, match_to_enum, retry_importer, save_as_profile
+from src.gui.importer.common import (
+    fix_offhand_type,
+    fix_weapon_type,
+    get_class_name,
+    get_with_retry,
+    match_to_enum,
+    retry_importer,
+    save_as_profile,
+)
 from src.item.data.affix import Affix
 from src.item.data.item_type import ItemType
 from src.item.descr.text import clean_str, closest_match
@@ -40,16 +48,16 @@ def import_mobalytics(url: str):
     Logger.info(f"Loading {url}")
     try:
         r = get_with_retry(url=url)
-    except ConnectionError as ex:
-        Logger.error(msg := "Couldn't get build")
-        raise MobalyticsException(msg) from ex
+    except ConnectionError as exc:
+        Logger.exception(msg := "Couldn't get build")
+        raise MobalyticsException(msg) from exc
     data = lxml.html.fromstring(r.text)
     build_elem = data.xpath(BUILD_GUIDE_NAME_XPATH)
     if not build_elem:
         Logger.error(msg := "No build found")
         raise MobalyticsException(msg)
     build_name = build_elem[0].tail
-    class_name = _get_class_name(input_str=build_elem[0].text)
+    class_name = get_class_name(input_str=build_elem[0].text)
     if not (stats_grid := data.xpath(STATS_GRID_XPATH)):
         Logger.error(msg := "No stats grid found")
         raise MobalyticsException(msg)
@@ -143,22 +151,6 @@ def _fix_input_url(url: str) -> str:
     return urlunparse((parsed_url.scheme, parsed_url.netloc, parsed_url.path, parsed_url.params, new_query_string, parsed_url.fragment))
 
 
-def _get_class_name(input_str: str) -> str:
-    input_str = input_str.lower()
-    if "barbarian" in input_str:
-        return "Barbarian"
-    if "druid" in input_str:
-        return "Druid"
-    if "necromancer" in input_str:
-        return "Necromancer"
-    if "rogue" in input_str:
-        return "Rogue"
-    if "sorcerer" in input_str:
-        return "Sorcerer"
-    Logger.error(f"Couldn't match class name {input_str=}")
-    return "Unknown"
-
-
 if __name__ == "__main__":
     Logger.init("debug")
     os.chdir(pathlib.Path(__file__).parent.parent.parent.parent)