initial commit

.vscode/argv.json

@@ -0,0 +1,24 @@
+// This configuration file allows you to pass permanent command line arguments to VS Code.
+// Only a subset of arguments is currently supported to reduce the likelihood of breaking
+// the installation.
+//
+// PLEASE DO NOT CHANGE WITHOUT UNDERSTANDING THE IMPACT
+//
+// NOTE: Changing this file requires a restart of VS Code.
+{
+	// Use software rendering instead of hardware accelerated rendering.
+	// This can help in cases where you see rendering issues in VS Code.
+	// "disable-hardware-acceleration": true,
+
+	// Enabled by default by VS Code to resolve color issues in the renderer
+	// See https://github.com/microsoft/vscode/issues/51791 for details
+	"disable-color-correct-rendering": true,
+
+	// Allows to disable crash reporting.
+	// Should restart the app if the value is changed.
+	"enable-crash-reporter": true,
+
+	// Unique id used for correlating crash reports sent from this instance.
+	// Do not edit this value.
+	"crash-reporter-id": "a5ac00eb-a84a-468e-9523-cfe30e06fb4b"
+}

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "args":["~/shared/Videos/Youtube-WL", "~/shared/Videos/Youtube-WL/playlist.json"],
+            "console": "integratedTerminal"
+        }
+    ]
+}

README.md

@@ -0,0 +1,29 @@
+# Installation
+
+1. install venv:
+Adapt the  version number to your python version accordingly
+```
+sudo apt-get install ffmpeg
+sudo apt install python3.9-venv
+python3 -m venv ytdlwl
+source ytdlwl/bin/activate
+./install-yt-dlp.sh
+```
+
+2. setup cron job
+```
+crontab -e
+```
+Add the following line to start the update daily at 3:00 am
+```
+0 3 * * * /usr/bin/flock -w 0 ~/ytdlwl/update_wl.lock ~/ytdlwl/update_wl.sh > ~/ytdlwl/update_wl.log 2>&1
+```
+
+flock prevents overlapping runs. For a daily job it might not be necessary, but it is also useful for testing purposes (just write * * * * *, so start it once a minute).
+
+# Update cookies
+The yt-dlp logsin to youtube using cookies:
+1. Start firefox and install addon "export cookies"
+2. login to youtube and select the channel to access
+3. Export cookies (of all sites, just to be sure).
+4. Save as ~/ytdlwl/cookies/cookies.txt

install-yt-dlp.sh

@@ -0,0 +1 @@
+python3 -m pip install --upgrade yt-dlp

remove_videos_not_in_playlist.py

@@ -0,0 +1,45 @@
+"""
+loads a json representation of a playlist as it was returned by yt-dlp -J and 
+checks if there is any file in the specified folder, whose id (the part in square brackets in the filename)
+is not part of the playlist. If so the file is deleted.
+"""
+
+import sys
+import json
+from pprint import pprint
+import os
+import re
+
+if len(sys.argv) != 3:
+    raise ValueError('usage: remove_videos_not_in_playlist <folder> <playlist-json-file>') 
+
+folder = sys.argv[1]
+jsonfile = sys.argv[2]
+
+with open(jsonfile) as f:
+    d = json.load(f)
+
+entries = d["entries"]
+ids = []
+for entry in entries:
+    try:
+        ids.append(entry["id"])
+    except:
+        # sometimes null or empty entries in playlist (maybe deleted videos)
+        # just skip
+        pass    
+
+# pprint(d)
+print(ids)
+
+files = os.listdir(folder)
+for file in files:
+    print(file)
+    m = re.match(".*\[([a-zA-Z0-9_\-]+)\].*", file)
+    if m is not None:
+        fileId = m[1]
+        print(fileId)
+        if not fileId in ids:
+            fullpath = os.path.join(folder, file)
+            print("Deleting " + fullpath + "...")
+            os.remove(fullpath)

set_file_date.py

@@ -0,0 +1,49 @@
+"""
+loads a json representation of a playlist as it was returned by yt-dlp -J.
+For all files in <folder> which have an id (in square brackets) in the filename,
+file modification and access time is set to the video upload date.
+"""
+
+import sys
+import json
+from pprint import pprint
+import os
+import re
+from datetime import datetime
+
+if len(sys.argv) != 3:
+    raise ValueError('usage: set_file_date <folder> <playlist-json-file>') 
+
+folder = sys.argv[1]
+jsonfile = sys.argv[2]
+
+with open(jsonfile) as f:
+    d = json.load(f)
+
+entries = d["entries"]
+ids = {}
+for entry in entries:
+	try:
+		id = entry["id"]
+		print("id=" + id)
+		print("upload_date=" + str(entry["upload_date"]))
+		ids[id] = datetime.strptime(entry["upload_date"],"%Y%m%d")
+	except:
+		print("failed to get datetime")
+
+# pprint(d)
+print(ids)
+
+files = os.listdir(folder)
+for file in files:
+    print(file)
+    m = re.match(".*\[([a-zA-Z0-9_\-]+)\].*", file)
+    if m is not None:
+        fileId = m[1]
+        print(fileId)
+        fullpath = os.path.join(folder, file)
+        if  fileId in ids.keys():
+            dt = ids[fileId]
+            print("change time to " + str(dt))
+            os.utime(fullpath, (dt.timestamp(), dt.timestamp()))
+

update_wl.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+cd ~/ytdlwl
+source ~/ytdlwl/ytdlwl/bin/activate
+
+OUTPUT_PATH=~/shared/Videos/Youtube-WL
+
+echo downloading playlist...
+yt-dlp -J --cookies ~/ytdlwl/cookies/cookies.txt https://www.youtube.com/playlist?list=WL > ${OUTPUT_PATH}/playlist.json
+
+echo purging...
+python3 remove_videos_not_in_playlist.py ${OUTPUT_PATH} ${OUTPUT_PATH}/playlist.json
+
+echo downloading files...
+yt-dlp --embed-thumbnail --embed-metadata --embed-chapters --restrict-filenames -f "best[height=720]" --cookies ~/ytdlwl/cookies/cookies.txt --paths $OUTPUT_PATH https://www.youtube.com/playlist?list=WL 
+#yt-dlp --embed-thumbnail --embed-metadata --embed-chapters --cookies ~/ytdlwl/cookies/cookies.txt --paths $OUTPUT_PATH https://www.youtube.com/playlist?list=WL 
+#yt-dlp --embed-thumbnail --embed-metadata --embed-chapters -f "bv*[height<=1080]+ba/b[height<=1080] / wv*+ba/w" --cookies ~/ytdlwl/cookies/cookies.txt --paths $OUTPUT_PATH https://www.youtube.com/playlist?list=WL 
+#yt-dlp --embed-thumbnail --embed-metadata --embed-chapters -S 'codec:h264' --cookies ~/ytdlwl/cookies/cookies.txt --paths $OUTPUT_PATH https://www.youtube.com/playlist?list=WL 
+
+echo adjusting file dates...
+python3 set_file_date.py ${OUTPUT_PATH} ${OUTPUT_PATH}/playlist.json