Merge branch 'unstable'

.github/workflows/release.yaml

@@ -29,6 +29,6 @@ jobs:
         run: python setup.py sdist bdist_wheel
       - name: Upload to PyPI
         env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: twine upload dist/* --skip-existing
+          TWINE_USERNAME: "__token__"
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: twine upload dist/*

README.md

@@ -169,7 +169,7 @@ usage: ytmdl [-h] [-q] [-o OUTPUT_DIR] [--song SONG-METADATA]
              [--filename NAME] [--pl-start NUMBER] [--pl-end NUMBER]
              [--pl-items ITEM_SPEC] [--ignore-errors] [--title-as-name]
              [--level LEVEL] [--disable-file] [--list-level]
-             [SONG_NAME [SONG_NAME ...]]
+             [SONG_NAME ...]
 
 positional arguments:
   SONG_NAME             Name of the song to download. Can be an URL to a
@@ -251,7 +251,8 @@ Metadata:
   --on-meta-error ON_META_ERROR
                         What to do if adding the metadata fails for some
                         reason like lack of metadata or perhaps a network
-                        issue. Options are ['exit', 'skip', 'manual']
+                        issue. Options are ['exit', 'skip', 'manual',
+                        'youtube']
 
 Playlist:
   --pl-start NUMBER     Playlist video to start at (default is 1)

setup.py

@@ -17,7 +17,7 @@
     'requests',
     'colorama',
     'beautifulsoup4',
-    'downloader-cli',
+    'downloader-cli>=0.3.4',
     'pyxdg',
     'ffmpeg-python',
     'pysocks',

ytmdl/__version__.py

@@ -1,2 +1,2 @@
 # Store the version of the package
-__version__ = "2023.07.27"
+__version__ = "2023.11.26"

ytmdl/core.py

@@ -16,6 +16,7 @@
 from ytmdl.exceptions import (
     DownloadError, ConvertError, NoMetaError, MetadataError
 )
+from ytmdl.meta.yt import extract_meta_from_yt
 
 
 logger = Logger("core")
@@ -200,7 +201,7 @@ def trim(name: str, args) -> None:
     trim.Trim(name)
 
 
-def meta(conv_name: str, song_name: str, search_by: str, args):
+def meta(conv_name: str, song_name: str, search_by: str, link: str, args):
     """Handle adding the metadata for the passed song.
 
     We will use the passed name to search for metadata, ask
@@ -236,14 +237,26 @@ def meta(conv_name: str, song_name: str, search_by: str, args):
 
     # If no meta was found raise error
     if not TRACK_INFO:
-        # Check if we are supposed to add manual meta
-        if args.on_meta_error != "manual":
+        # Check if we are supposed to add manual meta or from youtube
+        if args.on_meta_error not in ["manual", "youtube"]:
             raise NoMetaError(search_by)
-
-        TRACK_INFO = manual.get_data(song_name)
-        return TRACK_INFO
+
+        if args.on_meta_error == "manual":
+            TRACK_INFO = manual.get_data(song_name)
+        elif args.on_meta_error == 'youtube':
+            # Extract meta from youtube
+            track_info = extract_meta_from_yt(link)
+            TRACK_INFO = [track_info]
+
+        option = song.setData(TRACK_INFO, IS_QUIET, conv_name, PASSED_FORMAT, 0, skip_showing_choice=True)
+        if not isinstance(option, int):
+            raise MetadataError(search_by)
+
+        return TRACK_INFO[option]
 
     logger.info('Setting data...')
+
+
     option = song.setData(TRACK_INFO, IS_QUIET, conv_name, PASSED_FORMAT,
                           args.choice)
 
@@ -260,6 +273,6 @@ def meta(conv_name: str, song_name: str, search_by: str, args):
         logger.info(
             "Amending the search because -2 was entered as the option")
         search_by = utility.get_new_meta_search_by(search_by)
-        return meta(conv_name, song_name, search_by, args)
+        return meta(conv_name, song_name, search_by, link, args)
 
     return TRACK_INFO[option]

ytmdl/dir.py

@@ -22,6 +22,17 @@ def __replace_special_characters(passed_name: str) -> str:
     return sub(r'/', '-', passed_name)
 
 
+def get_abs_path(path_passed: str) -> str:
+    """
+    Get the absolute path by removing special path directives
+    that `ytmdl` supports.
+    """
+    if "$" not in path_passed:
+        return path_passed
+
+    return path_passed.split("$")[0]
+
+
 def cleanup(TRACK_INFO, index, datatype, remove_cached=True, filename_passed=None):
     """Move the song from temp to the song dir."""
     try:
@@ -42,7 +53,7 @@ def cleanup(TRACK_INFO, index, datatype, remove_cached=True, filename_passed=Non
             SONG_NAME = filename_passed + ".{}".format(datatype)
 
         DIR = defaults.DEFAULT.SONG_DIR
-        logger.debug(DIR)
+        logger.debug("directory being used: ", DIR)
 
         # Check if DIR has $ in its path
         # If it does then make those folders accordingly

ytmdl/main.py

@@ -411,7 +411,7 @@ def post_processing(
 
     # Else fill the meta by searching
     try:
-        track_selected = meta(conv_name, song_name, song_metadata, args)
+        track_selected = meta(conv_name, song_name, song_metadata, link, args)
     except NoMetaError as no_meta_error:
         if args.on_meta_error == 'skip':
             # Write to the archive file
@@ -493,7 +493,7 @@ def pre_checks(args):
 
     # Ensure the output directory is legitimate
     if (args.output_dir is not None):
-        if path.isdir(path.expanduser(args.output_dir)):
+        if path.isdir(path.expanduser(dir.get_abs_path(args.output_dir))):
             defaults.DEFAULT.SONG_DIR = path.expanduser(args.output_dir)
         else:
             logger.warning(

ytmdl/manual.py

@@ -26,14 +26,24 @@ class Meta:
     track_number        : Number of the track in the album
     artwork_url_100     : URL of the album cover
     """
-    def __init__(self):
-        self.release_date = "{}T00:00:00Z".format(datetime.now().date())
-        self.track_name = "N/A"
-        self.artist_name = "N/A"
-        self.collection_name = "N/A"
-        self.primary_genre_name = "N/A"
-        self.track_number = "1"
-        self.artwork_url_100 = ""
+    def __init__(
+        self,
+        release_date: str = None,
+        track_name: str = "N/A",
+        artist_name: str = "N/A",
+        collection_name: str = "N/A",
+        primary_genre_name: str = "N/A",
+        track_number: str = "1",
+        artwork_url_100: str = ""
+    ):
+        self.release_date = "{}T00:00:00Z".format(datetime.now().date()) if \
+            release_date is None else release_date
+        self.track_name = track_name
+        self.artist_name = artist_name
+        self.collection_name = collection_name
+        self.primary_genre_name = primary_genre_name
+        self.track_number = track_number
+        self.artwork_url_100 = artwork_url_100
 
     def _read_individual(self, default_value):
         """

ytmdl/meta/yt.py

@@ -0,0 +1,54 @@
+"""
+Handle metadata extraction from youtube
+"""
+
+from typing import Dict, List
+from datetime import datetime
+
+from yt_dlp import YoutubeDL
+from simber import Logger
+
+from ytmdl.manual import Meta
+from ytmdl.utils.ytdl import get_ytdl_opts
+from ytmdl.exceptions import ExtractError
+
+# Create logger
+logger = Logger("meta:yt")
+
+
+def __parse_release_date_from_utc(timestamp: int) -> str:
+    if timestamp is None:
+        return None
+
+    dt_object = datetime.utcfromtimestamp(timestamp)
+    return dt_object.strftime('%Y-%m-%dT%H:%M:%SZ')
+
+def __parse_genre_name_from_categories(categories: List[str]) -> str:
+    return categories[0] if len(categories) else "N/A"
+
+def __parse_meta_from_details(details: Dict) -> Meta:
+    """
+    Parse the meta object from the passed details
+    """
+    return Meta(
+        release_date=__parse_release_date_from_utc(details.get("release_timestamp", None)),
+        track_name=details.get("title", "N/A"),
+        artist_name=details.get("channel", "N/A"),
+        primary_genre_name=__parse_genre_name_from_categories(details.get("categories", [])),
+        artwork_url_100=details.get("thumbnail", "N/A")
+    )
+
+def extract_meta_from_yt(video_url: str) -> Meta:
+    """
+    Extract the metadata from the passed video ID and return
+    it accordingly.
+    """
+    ytdl_obj = YoutubeDL(get_ytdl_opts())
+
+    try:
+        details = ytdl_obj.extract_info(video_url, download=False)
+        return __parse_meta_from_details(details)
+    except Exception as e:
+        logger.debug("Got exception while extracting details for video: ", video_url)
+        logger.warning("Failed to extract metadata from yt with exception: ", str(e))
+        raise ExtractError(f"error extracting data from yt: {str(e)}")

ytmdl/setupConfig.py

@@ -139,7 +139,7 @@ def __init__(self):
 
         self.DEFAULT_FORMAT = 'mp3'
 
-        self.ON_ERROR_OPTIONS = ['exit', 'skip', 'manual']
+        self.ON_ERROR_OPTIONS = ['exit', 'skip', 'manual', 'youtube']
 
         self.ON_ERROR_DEFAULT = 'exit'
 

ytmdl/song.py

@@ -384,15 +384,19 @@ def _get_option(SONG_INFO, is_quiet, choice):
     return int(option)
 
 
-def setData(SONG_INFO, is_quiet, song_path, datatype='mp3', choice=None):
+def setData(SONG_INFO, is_quiet, song_path, datatype='mp3', choice=None, skip_showing_choice: bool = False):
     """Add the metadata to the song."""
 
     # Some providers need extra daa from other endpoints,
     # this is where we define which need it and where to get
     # it from
 
     logger.debug(choice)
-    option = _get_option(SONG_INFO, is_quiet, choice)
+    option = choice
+
+    if not skip_showing_choice:
+        option = _get_option(SONG_INFO, is_quiet, choice)
+
     logger.debug(option)
 
     # If -1 or -2 then skip setting the metadata

ytmdl/stringutils.py

@@ -57,7 +57,7 @@ def compute_jaccard(tokens1, tokens2):
     return len(intersect)/len(union)
 
 def remove_unwanted_chars(string):
-    return re.sub(r" |/", "#", string)
+    return re.sub(r"\s|\||/", "#", string)
 
 
 def urlencode(text):

ytmdl/utils/ytdl.py

@@ -14,6 +14,20 @@
 logger = Logger("yt")
 
 
+def get_ytdl_opts() -> Dict:
+    is_quiet: bool = utility.determine_logger_level(
+        ) != logger.level_map["DEBUG"]
+    no_warnings: bool = utility.determine_logger_level(
+        ) > logger.level_map["WARNING"]
+
+    return {
+        "quiet": is_quiet,
+        'no_warnings': no_warnings,
+        'nocheckcertificate': True,
+        'source_address': '0.0.0.0'
+    }
+
+
 def is_ytdl_config_present(path_passed: str) -> bool:
     """
     Check if the passed file is present or not.
@@ -40,17 +54,7 @@ def ydl_opts_with_config(ytdl_config: str = None) -> Dict:
 
     If the config is not present, return an empty dictionary
     """
-    is_quiet: bool = utility.determine_logger_level(
-    ) != logger.level_map["DEBUG"]
-    no_warnings: bool = utility.determine_logger_level(
-    ) > logger.level_map["WARNING"]
-
-    ydl_opts = {
-        "quiet": is_quiet,
-        'no_warnings': no_warnings,
-        'nocheckcertificate': True,
-        'source_address': '0.0.0.0'
-    }
+    ydl_opts = get_ytdl_opts()
 
     # If config is passed, generated opts with config
     if ytdl_config is not None:

ytmdl/yt.py

@@ -37,7 +37,8 @@ def get_youtube_streams(url):
 
 # Function to be called by ytdl progress hook.
 def progress_handler(d):
-    d_obj = Download('', '')
+    d_obj = Download('', '', icon_done="━", icon_left="━",
+                     icon_current=" ", color_done="green", color_left="black", icon_border=" ")
 
     if d['status'] == 'downloading':
         length = d_obj._get_terminal_length()