Title template (snobu#194)

* added template option and validation

* update comment link to element list

* get author info when fetching video info

* added template elements to video object

* minor function naming changes

* better exit message for template error

* changed template elements for better substitution

* implemented video title template

* removed trailing decimals on duration

* added template description

* removed hashing from uniqueId
removed debug logger.warn()

* fixed typos in default template
added elements to template fail message

* moved ffmpeg version logging to verbose

README.md

@@ -19,6 +19,11 @@ This release would not have been possible without the code and time contributed
 - [Università della Calabria][unical]: fork over at https://github.com/peppelongo96/UnicalDown
 
 ## What's new
+### v2.2
+
+- Added title template
+
+### v2.1
 
 - Major code refactoring (all credits to @lukaarma)
 - Destreamer is now able to refresh the session's access token. Use this with `-k` (keep cookies) and tick "Remember Me" on login.
@@ -138,6 +143,26 @@ https://web.microsoftstream.com/video/xxxxxxxx-aaaa-xxxx-xxxx-xxxxxxxxxxxx
  -dir=videos/lessons/week2"
 ```
 
+### Title template
+The `-t` option allows users to input a template string for the output file names.
+In the template you have to use 1 or more of the following special sequence that will be substituted at runtime.
+The special sequences must be surrounded by curly brackets like this '{title} {publishDate}'
+
+- `title`: the video title
+- `duration`: the video duration in HH:MM:SS format
+- `publishDate`: the date when the video was first published in YYYY-MM-DD format
+- `publishTime`: the time when the video was first published in HH:MM:SS format
+- `author`: the video publisher's name
+- `authorEmail`: the video publisher's email
+- `uniqueId`: a (almost) unique ID generated from the video informations
+
+Example -
+```
+Input:
+-t '{title} - {duration} - {publishDate} - {publishTime} - {author} - {authorEmail} - {uniqueId}'
+Expected filename:
+This is an example - 0:16:18 - 2020-07-30 - 10:30:13 - lukaarma - [email protected] - #3c6ca929.mkv
+```
 
 ## Expected output
 

src/CommandLineParser.ts

@@ -1,9 +1,11 @@
 import { CLI_ERROR, ERROR_CODE } from './Errors';
 import { checkOutDir } from './Utils';
 import { logger } from './Logger';
+import { templateElements } from './Types';
 
 import fs from 'fs';
 import readlineSync from 'readline-sync';
+import sanitize from 'sanitize-filename';
 import yargs from 'yargs';
 
 
@@ -33,6 +35,13 @@ export const argv: any = yargs.options({
  default: 'videos',
  demandOption: false
  },
+ outputTemplate: {
+ alias: 't',
+ describe: 'The template for the title. See the README for more info.',
+ type: 'string',
+ default: '{title} - {publishDate} {uniqueId}',
+ demandOption: false
+ },
  keepLoginCookies: {
  alias: 'k',
  describe: 'Let Chromium cache identity provider cookies so you can use "Remember me" during login',
@@ -102,7 +111,7 @@ export const argv: any = yargs.options({
 })
 .wrap(120)
 .check(() => noArguments())
-.check((argv: any) => inputConflicts(argv.videoUrls, argv.inputFile))
+.check((argv: any) => checkInputConflicts(argv.videoUrls, argv.inputFile))
 .check((argv: any) => {
  if (checkOutDir(argv.outputDirectory)) {
  return true;
@@ -113,6 +122,7 @@ export const argv: any = yargs.options({
  throw new Error(' ');
  }
 })
+.check((argv: any) => isOutputTemplateValid(argv))
 .argv;
 
 
@@ -129,7 +139,7 @@ function noArguments(): boolean {
 }
 
 
-function inputConflicts(videoUrls: Array<string | number> | undefined,
+function checkInputConflicts(videoUrls: Array<string | number> | undefined,
  inputFile: string | undefined): boolean {
  // check if both inputs are declared
  if ((videoUrls !== undefined) && (inputFile !== undefined)) {
@@ -162,6 +172,39 @@ function inputConflicts(videoUrls: Array<string | number> | undefined,
 }
 
 
+function isOutputTemplateValid(argv: any): boolean {
+ let finalTemplate: string = argv.outputTemplate;
+ const elementRegEx = RegExp(/{(.*?)}/g);
+ let match = elementRegEx.exec(finalTemplate);
+
+ // if no template elements this fails
+ if (match) {
+ // keep iterating untill we find no more elements
+ while (match) {
+ if (!templateElements.includes(match[1])) {
+ logger.error(
+ `'${match[0]}' is not aviable as a template element \n` +
+ `Aviable templates elements: '${templateElements.join("', '")}' \n`,
+ { fatal: true }
+ );
+
+ process.exit(1);
+ }
+ match = elementRegEx.exec(finalTemplate);
+ }
+ }
+ // bad template from user, switching to default
+ else {
+ logger.warn('Empty output template provided, using default one \n');
+ finalTemplate = '{title} - {publishDate} {uniqueId}';
+ }
+
+ argv.outputTemplate = sanitize(finalTemplate.trim());
+
+ return true;
+}
+
+
 export function promptUser(choices: Array<string>): number {
  let index: number = readlineSync.keyInSelect(choices, 'Which resolution/format do you prefer?');
 

src/Types.ts

@@ -6,11 +6,29 @@ export type Session = {
 
 
 export type Video = {
- date: string;
  title: string;
+ duration: string;
+ publishDate: string;
+ publishTime: string;
+ author: string;
+ authorEmail: string;
+ uniqueId: string;
  outPath: string;
  totalChunks: number; // Abstraction of FFmpeg timemark
  playbackUrl: string;
  posterImageUrl: string;
  captionsUrl?: string
 }
+
+
+/* TODO: expand this template once we are all on board with a list
+see https://github.com/snobu/destreamer/issues/190#issuecomment-663718010 for list*/
+export const templateElements: Array<string> = [
+ 'title',
+ 'duration',
+ 'publishDate',
+ 'publishTime',
+ 'author',
+ 'authorEmail',
+ 'uniqueId'
+];

src/Utils.ts

@@ -182,7 +182,7 @@ export function checkOutDir(directory: string): boolean {
 export function checkRequirements(): void {
  try {
  const ffmpegVer: string = execSync('ffmpeg -version').toString().split('\n')[0];
- logger.info(`Using ${ffmpegVer}\n`);
+ logger.verbose(`Using ${ffmpegVer}\n`);
  }
  catch (e) {
  process.exit(ERROR_CODE.MISSING_FFMPEG);

src/VideoUtils.ts

@@ -5,10 +5,9 @@ import { Video, Session } from './Types';
 
 import { AxiosResponse } from 'axios';
 import fs from 'fs';
-import { parse } from 'iso8601-duration';
+import { parse as parseDuration, Duration } from 'iso8601-duration';
 import path from 'path';
-import sanitize from 'sanitize-filename';
-
+import sanitizeWindowsName from 'sanitize-filename';
 
 function publishedDateToString(date: string): string {
  const dateJs: Date = new Date(date);
@@ -19,8 +18,25 @@ function publishedDateToString(date: string): string {
 }
 
 
+function publishedTimeToString(date: string): string {
+ const dateJs: Date = new Date(date);
+ const hours: string = dateJs.getHours().toString();
+ const minutes: string = dateJs.getMinutes().toString();
+ const seconds: string = dateJs.getSeconds().toString();
+
+ return `${hours}:${minutes}:${seconds}`;
+}
+
+
+function isoDurationToString(time: string): string {
+ const duration: Duration = parseDuration(time);
+
+ return `${duration.hours ?? '00'}:${duration.minutes ?? '00'}:${duration.seconds?.toFixed(0) ?? '00'}`;
+}
+
+
 function durationToTotalChunks(duration: string): number {
- const durationObj: any = parse(duration);
+ const durationObj: any = parseDuration(duration);
  const hrs: number = durationObj.hours ?? 0;
  const mins: number = durationObj.minutes ?? 0;
  const secs: number = Math.ceil(durationObj.seconds ?? 0);
@@ -32,18 +48,42 @@ function durationToTotalChunks(duration: string): number {
 export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
  let metadata: Array<Video> = [];
  let title: string;
- let date: string;
+ let duration: string;
+ let publishDate: string;
+ let publishTime: string;
+ let author: string;
+ let authorEmail: string;
+ let uniqueId: string;
+ const outPath = '';
  let totalChunks: number;
  let playbackUrl: string;
  let posterImageUrl: string;
  let captionsUrl: string | undefined;
 
  const apiClient: ApiClient = ApiClient.getInstance(session);
 
- for (const GUID of videoGuids) {
- let response: AxiosResponse<any> | undefined= await apiClient.callApi('videos/' + GUID, 'get');
+ /* TODO: change this to a single guid at a time to ease our footprint on the
+ MSS servers or we get throttled after 10 sequential reqs */
+ for (const guid of videoGuids) {
+ let response: AxiosResponse<any> | undefined =
+ await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
+
+ title = sanitizeWindowsName(response?.data['name']);
+
+ duration = isoDurationToString(response?.data.media['duration']);
+
+ publishDate = publishedDateToString(response?.data['publishedDate']);
+
+ publishTime = publishedTimeToString(response?.data['publishedDate']);
+
+ author = response?.data['creator'].name;
+
+ authorEmail = response?.data['creator'].mail;
+
+ uniqueId = '#' + guid.split('-')[0];
+
+ totalChunks = durationToTotalChunks(response?.data.media['duration']);
 
- title = sanitize(response?.data['name']);
  playbackUrl = response?.data['playbackUrls']
  .filter((item: { [x: string]: string; }) =>
  item['mimeType'] == 'application/vnd.apple.mpegurl')
@@ -52,11 +92,9 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
  })[0];
 
  posterImageUrl = response?.data['posterImage']['medium']['url'];
- date = publishedDateToString(response?.data['publishedDate']);
- totalChunks = durationToTotalChunks(response?.data.media['duration']);
 
  if (subtitles) {
- let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${GUID}/texttracks`, 'get');
+ let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
 
  if (!captions?.data.value.length) {
  captionsUrl = undefined;
@@ -74,10 +112,15 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
  }
 
  metadata.push({
- date: date,
- totalChunks: totalChunks,
  title: title,
- outPath: '',
+ duration: duration,
+ publishDate: publishDate,
+ publishTime: publishTime,
+ author: author,
+ authorEmail: authorEmail,
+ uniqueId: uniqueId,
+ outPath: outPath,
+ totalChunks: totalChunks, // Abstraction of FFmpeg timemark
  playbackUrl: playbackUrl,
  posterImageUrl: posterImageUrl,
  captionsUrl: captionsUrl
@@ -88,18 +131,29 @@ export async function getVideoInfo(videoGuids: Array<string>, session: Session,
 }
 
 
-export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, format: string, skip?: boolean): Array<Video> {
+export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, template: string, format: string, skip?: boolean): Array<Video> {
 
  videos.forEach((video: Video, index: number) => {
- let title = `${video.title} - ${video.date}`;
+ let title: string = template;
+ let finalTitle: string;
+ const elementRegEx = RegExp(/{(.*?)}/g);
+ let match = elementRegEx.exec(template);
+
+ while (match) {
+ let value = video[match[1] as keyof Video] as string;
+ title = title.replace(match[0], value);
+ match = elementRegEx.exec(template);
+ }
+
  let i = 0;
+ finalTitle = title;
 
- while (!skip && fs.existsSync(path.join(outDirs[index], title + '.' + format))) {
- title = `${video.title} - ${video.date}_${++i}`;
+ while (!skip && fs.existsSync(path.join(outDirs[index], finalTitle + '.' + format))) {
+ finalTitle = `${title}.${++i}`;
  }
 
 
- video.outPath = path.join(outDirs[index], title + '.' + format);
+ video.outPath = path.join(outDirs[index], finalTitle + '.' + format);
  });
 
  return videos;

src/destreamer.ts

@@ -124,15 +124,15 @@ async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array
  logger.info('Fetching videos info... \n');
  const videos: Array<Video> = createUniquePath (
  await getVideoInfo(videoGUIDs, session, argv.closedCaptions),
- outputDirectories, argv.format, argv.skip
+ outputDirectories, argv.outputTemplate, argv.format, argv.skip
  );
 
  if (argv.simulate) {
  videos.forEach((video: Video) => {
  logger.info(
  '\nTitle: '.green + video.title +
  '\nOutPath: '.green + video.outPath +
- '\nPublished Date: '.green + video.date +
+ '\nPublished Date: '.green + video.publishDate +
  '\nPlayback URL: '.green + video.playbackUrl +
  ((video.captionsUrl) ? ('\nCC URL: '.green + video.captionsUrl) : '')
  );