[#11] Support another services: common logic fixes

src/config/app.py

@@ -56,3 +56,5 @@ def members(cls) -> list["SupportedService"]:
 TMP_DATA_DIR = PROJECT_PATH.parent / ".data"
 
 DEBUG_SHUTDOWNS = os.getenv("DEBUG_SHUTDOWNS", "false").lower() == "true"
+
+SSL_REQUEST_VERIFY = os.getenv("SSL_REQUEST_VERIFY", "true").lower() == "true"

src/parsing/__init__.py

@@ -0,0 +1,2 @@
+from .main_parsing import BaseParser
+from .spb_services import *

src/parsing/main_parsing.py

@@ -1,7 +1,6 @@
 import abc
 import hashlib
 import logging
-import os
 import urllib.parse
 from datetime import datetime, timedelta, date, timezone
 from typing import ClassVar, Type
@@ -10,7 +9,13 @@
 
 from src.db.models import Address, DateRange
 from src.utils import ADDRESS_DEFAULT_PATTERN
-from src.config.app import RESOURCE_URLS, SupportedCity, SupportedService, DATA_PATH
+from src.config.app import (
+    RESOURCE_URLS,
+    SupportedCity,
+    SupportedService,
+    DATA_PATH,
+    SSL_REQUEST_VERIFY,
+)
 
 logger = logging.getLogger("parsing.main")
 
@@ -75,7 +80,7 @@ def cashed_filename(url: str) -> str:
             return tmp_file_path.read_text()
 
         logger.debug("Getting content for service: %s ...", url)
-        with httpx.Client() as client:
+        with httpx.Client(verify=SSL_REQUEST_VERIFY) as client:
             response = client.get(url)
             response_data = response.text
 
@@ -93,7 +98,7 @@ def _format_date(date: datetime | date) -> str:
 
     @staticmethod
     def _clear_string(src_string: str) -> str:
-        return src_string.replace("\n", "").replace(" ", "").strip()
+        return src_string.replace("\n", "").strip()
 
     @classmethod
     def get_parsers(cls) -> dict[SupportedService, Type["BaseParser"]]:

src/parsing/spb_services.py

@@ -63,7 +63,7 @@ def _parse_website(
         for row in rows:
             if row_streets := row.xpath(".//td[@class='rowStreets']"):
                 addresses = row_streets[0].xpath(".//span/text()")
-                dates = row.xpath("td/text()")[4:8]
+                dates = [td.text for td in row.xpath("td")[3:7]]
                 date_start, time_start, date_end, time_end = map(self._clear_string, dates)
 
                 if len(addresses) == 1:

src/providers/shutdowns.py

@@ -66,6 +66,7 @@ def for_addresses(cls, addresses: list[str]) -> list[ShutDownByServiceInfo]:
 
         """
         shutdown_info_list = []
+        # # for (service, parser_cls) in BaseParser.get_parsers().items():
         for service in SupportedService.members():
             for address in addresses:
                 if shutdowns := cls.for_address(SupportedCity.SPB, address, service):