feat(net): allow NFAT multi-mapping

- NFAT now enrolls reused files as well.
- Allow NFAT to keep multiple records for files with the same hash.
- Enable NFAT when downloading.

src/main/net/aria2.ts

@@ -15,6 +15,7 @@ import { net } from "electron";
 import path from "path";
 import { dlchk } from "@/main/net/dlchk";
 import fs from "fs-extra";
+import { nfat } from "@/main/net/nfat";
 
 let aria2cProcess: childProcess.ChildProcess | null = null;
 let aria2cToken: string | null = null;
@@ -24,6 +25,7 @@ let isAvailable = false;
 
 export interface Aria2DownloadRequest {
     urls: string[];
+    origin: string;
     path: string;
     sha1?: string;
     size?: number;
@@ -36,6 +38,11 @@ const taskResolvers = new Map<string, (r: boolean) => void>();
  * Preflights and resolves the given request.
  */
 async function resolve(req: Aria2DownloadRequest): Promise<[Promise<boolean>, string]> {
+    if (req.sha1) {
+        await nfat.deploy(req.path, req.origin, req.sha1);
+    }
+
+
     // Preflight
     const pref = await dlchk.validate({ ...req });
 
@@ -80,7 +87,12 @@ async function commit(req: Aria2DownloadRequest): Promise<[Promise<boolean>, str
     console.debug(`Committed aria2 task ${gid}`);
 
     const p = new Promise<boolean>((res) => {
-        taskResolvers.set(gid, res);
+        taskResolvers.set(gid, (b: boolean) => {
+            if (req.sha1) {
+                nfat.enroll(req.path, req.origin, req.sha1);
+            }
+            res(b);
+        });
     });
 
     return [p, gid];

src/main/net/dlx.ts

@@ -48,7 +48,11 @@ async function getAll(req: DlxDownloadRequest[], init?: DlxDownloadInit): Promis
     let promises: Promise<unknown>[] = [];
 
     if (dl === "aria2") {
-        const aria2Tasks: Aria2DownloadRequest[] = req.map(r => ({ ...r, urls: mirror.apply(r.url) }));
+        const aria2Tasks: Aria2DownloadRequest[] = req.map(r => ({
+            ...r,
+            urls: mirror.apply(r.url),
+            origin: r.url
+        }));
         const res = await Promise.all(aria2Tasks.map(t => aria2.resolve(t)));
 
         function cancelAllAria2() {
@@ -73,7 +77,11 @@ async function getAll(req: DlxDownloadRequest[], init?: DlxDownloadInit): Promis
 
 
     } else if (dl === "next") {
-        const nextTasks: NextDownloadRequest[] = req.map(r => ({ ...r, urls: mirror.apply(r.url) }));
+        const nextTasks: NextDownloadRequest[] = req.map(r => ({
+            ...r,
+            urls: mirror.apply(r.url),
+            origin: r.url
+        }));
 
         const handlers = nextdl.gets(nextTasks);
 

src/main/net/nextdl.ts

@@ -14,6 +14,7 @@ import EventEmitter from "events";
 import type TypedEmitter from "typed-emitter";
 import path from "node:path";
 import { dlchk } from "@/main/net/dlchk";
+import { nfat } from "@/main/net/nfat";
 
 /**
  * Events emitted when downloading.
@@ -41,6 +42,7 @@ export enum NextDownloadStatus {
  */
 export interface NextDownloadRequest {
     urls: string[];
+    origin: string;
     path: string;
     sha1?: string;
     size?: number;
@@ -123,6 +125,11 @@ function gets(req: NextDownloadRequest[]): [Promise<boolean>, NextDownloadTask][
  * Resolves the download task for the maximum number of tries specified.
  */
 async function resolve(task: NextDownloadTask): Promise<boolean> {
+    // First try to reuse existing files
+    if (task.req.sha1) {
+        await nfat.deploy(task.req.path, task.req.origin, task.req.sha1);
+    }
+
     // Preflight validate
     // For files that cannot be validated, re-downloading is suggested and therefore not skipped
     const valid = await dlchk.validate({ ...task.req }) === "checked";
@@ -144,6 +151,12 @@ async function resolve(task: NextDownloadTask): Promise<boolean> {
             }
             if (st === NextRequestStatus.SUCCESS) {
                 task.status = NextDownloadStatus.DONE;
+
+                // Add file for reusing
+                if (task.req.sha1) {
+                    nfat.enroll(task.req.path, task.req.origin, task.req.sha1);
+                }
+
                 return true;
             }
 

src/main/net/nfat.ts

@@ -7,6 +7,7 @@ import { paths } from "@/main/fs/paths";
 import { hash } from "@/main/security/hash";
 import fs from "fs-extra";
 import path from "path";
+import { conf } from "@/main/conf/conf";
 
 let db: Database;
 
@@ -31,9 +32,9 @@ async function init() {
     db.exec(`
         CREATE TABLE IF NOT EXISTS files
         (
-            sha1 VARCHAR(40) PRIMARY KEY NOT NULL,
-            url  TEXT                    NOT NULL,
-            path TEXT                    NOT NULL
+            sha1 VARCHAR(40) NOT NULL,
+            url  TEXT        NOT NULL,
+            path TEXT        NOT NULL
         );
     `);
 }
@@ -47,15 +48,19 @@ let removeStmt: Statement;
  * Adds a reusable file after download.
  */
 function enroll(fp: string, url: string, sha1: string) {
+    if (!conf().net.nfat.enable) return;
+
     if (!enrollStmt) {
         enrollStmt = db.prepare(`
-            INSERT INTO files
+            INSERT OR IGNORE INTO files
             VALUES (?, ?, ?);
         `);
         statements.push(enrollStmt);
     }
 
-    enrollStmt.run([sha1, url, fp]);
+    const pt = path.normalize(path.resolve(fp));
+
+    enrollStmt.run([sha1, url, pt]);
 }
 
 async function request(url: string, sha1: string): Promise<string | null> {
@@ -69,51 +74,55 @@ async function request(url: string, sha1: string): Promise<string | null> {
         statements.push(requestStmt);
     }
 
-    const r = requestStmt.get(sha1) as FATRecord | null;
+    const results = requestStmt.all(sha1) as unknown as FATRecord[];
 
-    if (!r || r.url !== url) return null;
+    for (const r of results) {
+        if (r.url !== url) continue;
 
-    try {
-        await fs.access(r.path);
-    } catch {
-        return null; // File might have been moved
-    }
+        try {
+            await fs.access(r.path);
 
-    try {
-        const fh = await hash.forFile(r.path, "sha1");
-        if (fh.toLowerCase() === sha1.toLowerCase()) {
-            // Hash validated, reuse this file
-            return r.path;
-        } else {
-            // Hash mismatch
-            if (!removeStmt) {
-                removeStmt = db.prepare(`
-                    DELETE
-                    FROM files
-                    WHERE sha1 = ?;
-                `);
-                statements.push(removeStmt);
+            const fh = await hash.forFile(r.path, "sha1");
+            if (fh.toLowerCase() === sha1.toLowerCase()) {
+                // Hash validated, reuse this file
+                return r.path;
             }
+        } catch {}
 
-            removeStmt.run(sha1);
-            return null;
-        }
-    } catch {
-        return null;
+        remove(r.path);
     }
+
+    return null;
+}
+
+function remove(pt: string) {
+    if (!removeStmt) {
+        removeStmt = db.prepare(`
+            DELETE
+            FROM files
+            WHERE path = ?;
+        `);
+        statements.push(removeStmt);
+    }
+
+    removeStmt.run(pt);
 }
 
 /**
  * Tries to find and reuse existing file by copying it to the new location. Returns whether the file
  * has been deployed.
  */
 async function deploy(target: string, url: string, sha1: string): Promise<boolean> {
+    if (!conf().net.nfat.enable) return false;
+
     const p = await request(url, sha1);
     if (!p) return false;
 
     try {
         await fs.ensureDir(path.dirname(target));
         await fs.copyFile(p, target);
+        enroll(target, url, sha1); // The copied file can also be reused
+        console.debug(`Reused ${p} -> ${target}`);
         return true;
     } catch (e) {
         console.warn(`Unable to reuse file ${p}: ${e}`);