Formatting

apis/api_helper.py

@@ -157,7 +157,7 @@ async def json_request(
         method: str = "GET",
         stream: bool = False,
         json_format: bool = True,
-        payload: Dict[str, str] = {},
+        payload: dict[str, str] = {},
     ) -> Any:
         headers = {}
         custom_session = False

apis/onlyfans/classes/create_auth.py

@@ -185,10 +185,12 @@ async def get_lists(self, refresh=True, limit=100, offset=0):
         self.lists = results
         return results
 
-    async def get_user(self, identifier: Union[str, int]) -> Union[create_user, dict]:
+    async def get_user(
+        self, identifier: Union[str, int]
+    ) -> Union[create_user, error_details]:
         link = endpoint_links(identifier).users
         response = await self.session_manager.json_request(link)
-        if isinstance(response, dict):
+        if not isinstance(response, error_details):
             response["session_manager"] = self.session_manager
             response = create_user(response, self)
         return response
@@ -257,7 +259,8 @@ async def get_subscriptions(
             self.pool = temp_pool
             self.paid_content = temp_paid_content
             temp_auth = await self.get_user(self.username)
-            json_authed = json_authed | temp_auth.__dict__
+            if isinstance(json_authed, dict):
+                json_authed = json_authed | temp_auth.__dict__
 
             subscription = create_user(json_authed, self)
             subscription.subscriber = self

classes/prepare_metadata.py

@@ -408,14 +408,14 @@ def __iter__(self):
         for attr, value in self.__dict__.items():
             yield attr, value
 
-    def reformat(self, unformatted_list) -> list[str]:
+    async def reformat(self, unformatted_list) -> list[str]:
         x = []
         format_variables2 = format_variables()
         for key, unformatted_item in unformatted_list.items():
             if "filename_format" == key:
                 unformatted_item = os.path.join(x[1], unformatted_item)
                 print
-            string = main_helper.reformat(self, unformatted_item)
+            string = await main_helper.reformat(self, unformatted_item)
             final_path = []
             paths = string.split(os.sep)
             for path in paths:

extras/OFRenamer/start.py → extras/OFRenamer/start_ofr.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
+import asyncio
 import os
+from random import randint
 import shutil
 import traceback
 import urllib.parse as urlparse
@@ -68,7 +70,7 @@ async def fix_directories2(post: api_table, media_db: list[media_table]):
             option["preview"] = media.preview
             option["archived"] = post.archived
             prepared_format = prepare_reformat(option)
-            file_directory = main_helper.reformat(
+            file_directory = await main_helper.reformat(
                 prepared_format, file_directory_format
             )
             prepared_format.directory = file_directory
@@ -87,7 +89,9 @@ async def fix_directories2(post: api_table, media_db: list[media_table]):
                 old_filepaths = [x for x in old_filepaths if "linked_" not in x]
             if old_filepaths:
                 old_filepath = old_filepaths[0]
-            new_filepath = main_helper.reformat(prepared_format, filename_format)
+            # a = randint(0,1)
+            # await asyncio.sleep(a)
+            new_filepath = await main_helper.reformat(prepared_format, filename_format)
             if old_filepath and old_filepath != new_filepath:
                 if os.path.exists(new_filepath):
                     os.remove(new_filepath)
@@ -133,7 +137,8 @@ async def fix_directories2(post: api_table, media_db: list[media_table]):
     result = database_session.query(folder.media_table)
     media_db = result.all()
     pool = api.pool
-    tasks = pool.starmap(fix_directories2, product(posts, [media_db]))
+    # tasks = pool.starmap(fix_directories2, product(posts, [media_db]))
+    tasks = [asyncio.ensure_future(fix_directories2(post,media_db)) for post in posts]
     settings = {"colour": "MAGENTA", "disable": False}
     delete_rows = await tqdm.gather(tasks, **settings)
     delete_rows = list(chain(*delete_rows))
@@ -186,7 +191,7 @@ async def start(
         reformats[key] = value.split(key2, 1)[0] + key2
         print
     print
-    a, base_directory, c = prepare_reformat(option, keep_vars=True).reformat(reformats)
+    a, base_directory, c = await prepare_reformat(option, keep_vars=True).reformat(reformats)
     download_info["base_directory"] = base_directory
     print
     all_files = []

helpers/main_helper.py

@@ -212,7 +212,7 @@ def legacy_database_fixer(database_path, database, database_name, database_exist
         export_sqlite(old_database_path, datas, database_name, legacy_fixer=True)
 
 
-def fix_sqlite(
+async def fix_sqlite(
     profile_directory,
     download_directory,
     metadata_directory,
@@ -229,7 +229,7 @@ def fix_sqlite(
         mandatory_directories["profile_directory"] = profile_directory
         mandatory_directories["download_directory"] = download_directory
         mandatory_directories["metadata_directory"] = metadata_directory
-        formatted_directories = format_directories(
+        formatted_directories = await format_directories(
             mandatory_directories,
             authed,
             site_name,
@@ -375,7 +375,7 @@ def format_paths(j_directories, site_name):
     return paths
 
 
-def reformat(prepared_format: prepare_reformat, unformatted):
+async def reformat(prepared_format: prepare_reformat, unformatted):
     post_id = prepared_format.post_id
     media_id = prepared_format.media_id
     date = prepared_format.date

modules/onlyfans.py

@@ -11,7 +11,7 @@
 from urllib.parse import urlparse
 
 import extras.OFLogin.start_ofl as oflogin
-import extras.OFRenamer.start as ofrenamer
+import extras.OFRenamer.start_ofr as ofrenamer
 import helpers.db_helper as db_helper
 import helpers.main_helper as main_helper
 from apis.onlyfans import onlyfans as OnlyFans
@@ -145,7 +145,7 @@ async def start_datascraper(
         username,
         metadata_directory_format,
     ]
-    fix_sqlite(*some_list)
+    await fix_sqlite(*some_list)
     api_array = scrape_choice(authed, subscription)
     api_array = format_options(api_array, "apis")
     apis = api_array[0]
@@ -275,7 +275,7 @@ async def profile_scraper(
     option["date_format"] = date_format
     option["maximum_length"] = text_length
     option["directory"] = base_directory
-    a, b, c = prepare_reformat(option, keep_vars=True).reformat(reformats)
+    a, b, c = await prepare_reformat(option, keep_vars=True).reformat(reformats)
     print
     y = await authed.get_subscription(identifier=model_username)
     override_media_types = []
@@ -368,7 +368,7 @@ async def paid_content_scraper(api: start, identifiers=[]):
                 mandatory_directories["profile_directory"] = profile_directory
                 mandatory_directories["download_directory"] = download_directory
                 mandatory_directories["metadata_directory"] = metadata_directory
-                formatted_directories = format_directories(
+                formatted_directories = await format_directories(
                     mandatory_directories,
                     authed,
                     site_name,
@@ -689,7 +689,7 @@ async def process_metadata(
                 os.remove(old_metadata)
 
 
-def format_directories(
+async def format_directories(
     directories: dict[str, Any],
     authed: create_auth,
     site_name: str,
@@ -725,7 +725,7 @@ def format_directories(
                 legacy_model_directory, "Metadata"
             )
         if key == "metadata_directory":
-            x["metadata_directory"] = main_helper.reformat(prepared_format, unformatted)
+            x["metadata_directory"] = await main_helper.reformat(prepared_format, unformatted)
     x["locations"] = []
     for location in locations:
         directories = {}
@@ -759,7 +759,7 @@ async def prepare_scraper(authed: create_auth, site_name, item):
     mandatory_directories["profile_directory"] = profile_directory
     mandatory_directories["download_directory"] = download_directory
     mandatory_directories["metadata_directory"] = metadata_directory
-    formatted_directories = format_directories(
+    formatted_directories = await format_directories(
         mandatory_directories,
         authed,
         site_name,
@@ -1147,11 +1147,11 @@ async def media_scraper(
             option["archived"] = new_post["archived"]
 
             prepared_format = prepare_reformat(option)
-            file_directory = main_helper.reformat(
+            file_directory = await main_helper.reformat(
                 prepared_format, file_directory_format
             )
             prepared_format.directory = file_directory
-            file_path = main_helper.reformat(prepared_format, filename_format)
+            file_path = await main_helper.reformat(prepared_format, filename_format)
             new_media["directory"] = os.path.join(file_directory)
             new_media["filename"] = os.path.basename(file_path)
             if file_directory not in directories:

requirements.txt

@@ -9,7 +9,7 @@ lxml
 mergedeep
 jsonpickle
 ujson
-sqlalchemy==1.4.15
+sqlalchemy==1.4.18
 alembic
 tqdm
 selenium