fix(server): external library sync not working for large libraries (i…

…mmich-app#7759)
zhaoterryy · Mar 11, 2024 · 5bd597f · 5bd597f
1 parent 49d9051
commit 5bd597f
Show file tree

Hide file tree

Showing 9 changed files with 106 additions and 104 deletions.
diff --git a/server/package-lock.json b/server/package-lock.json
diff --git a/server/package.json b/server/package.json
@@ -57,6 +57,7 @@
     "cookie-parser": "^1.4.6",
     "exiftool-vendored": "~24.5.0",
     "exiftool-vendored.pl": "12.76",
+    "fast-glob": "^3.3.2",
     "fluent-ffmpeg": "^2.1.2",
     "geo-tz": "^8.0.0",
     "glob": "^10.3.3",

diff --git a/server/src/domain/library/library.service.spec.ts b/server/src/domain/library/library.service.spec.ts
@@ -156,8 +156,7 @@ describe(LibraryService.name, () => {
 
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
       storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
-      assetMock.getPathsNotInLibrary.mockResolvedValue(['/data/user1/photo.jpg']);
-      assetMock.getByLibraryId.mockResolvedValue([]);
+      assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
 
       await sut.handleQueueAssetRefresh(mockLibraryJob);
 
@@ -183,7 +182,7 @@ describe(LibraryService.name, () => {
 
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
       storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
-      assetMock.getByLibraryId.mockResolvedValue([]);
+      assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
 
       await sut.handleQueueAssetRefresh(mockLibraryJob);
 
@@ -233,7 +232,7 @@ describe(LibraryService.name, () => {
 
       libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
       storageMock.crawl.mockResolvedValue([]);
-      assetMock.getByLibraryId.mockResolvedValue([]);
+      assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
 
       await sut.handleQueueAssetRefresh(mockLibraryJob);
 
@@ -242,6 +241,48 @@ describe(LibraryService.name, () => {
         exclusionPatterns: [],
       });
     });
+
+    it('should set missing assets offline', async () => {
+      const mockLibraryJob: ILibraryRefreshJob = {
+        id: libraryStub.externalLibrary1.id,
+        refreshModifiedFiles: false,
+        refreshAllFiles: false,
+      };
+
+      libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
+      storageMock.crawl.mockResolvedValue([]);
+      assetMock.getLibraryAssetPaths.mockResolvedValue({
+        items: [assetStub.image],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueAssetRefresh(mockLibraryJob);
+
+      expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.image.id], { isOffline: true });
+      expect(assetMock.updateAll).not.toHaveBeenCalledWith(expect.anything(), { isOffline: false });
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
+    });
+
+    it('should set crawled assets that were previously offline back online', async () => {
+      const mockLibraryJob: ILibraryRefreshJob = {
+        id: libraryStub.externalLibrary1.id,
+        refreshModifiedFiles: false,
+        refreshAllFiles: false,
+      };
+
+      libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
+      storageMock.crawl.mockResolvedValue([assetStub.offline.originalPath]);
+      assetMock.getLibraryAssetPaths.mockResolvedValue({
+        items: [assetStub.offline],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueAssetRefresh(mockLibraryJob);
+
+      expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.offline.id], { isOffline: false });
+      expect(assetMock.updateAll).not.toHaveBeenCalledWith(expect.anything(), { isOffline: true });
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
+    });
   });
 
   describe('handleAssetRefresh', () => {

diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts
@@ -640,27 +640,56 @@ export class LibraryService extends EventEmitter {
       .filter((validation) => validation.isValid)
       .map((validation) => validation.importPath);
 
-    const rawPaths = await this.storageRepository.crawl({
+    let rawPaths = await this.storageRepository.crawl({
       pathsToCrawl: validImportPaths,
       exclusionPatterns: library.exclusionPatterns,
     });
-    const crawledAssetPaths = rawPaths.map((filePath) => path.normalize(filePath));
+    const crawledAssetPaths = new Set<string>(rawPaths);
 
-    this.logger.debug(`Found ${crawledAssetPaths.length} asset(s) when crawling import paths ${library.importPaths}`);
+    const shouldScanAll = job.refreshAllFiles || job.refreshModifiedFiles;
+    let pathsToScan: string[] = shouldScanAll ? rawPaths : [];
+    rawPaths = [];
 
-    await this.assetRepository.updateOfflineLibraryAssets(library.id, crawledAssetPaths);
+    this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`);
 
-    if (crawledAssetPaths.length > 0) {
-      let filteredPaths: string[] = [];
-      if (job.refreshAllFiles || job.refreshModifiedFiles) {
-        filteredPaths = crawledAssetPaths;
-      } else {
-        filteredPaths = await this.assetRepository.getPathsNotInLibrary(library.id, crawledAssetPaths);
+    const assetIdsToMarkOffline = [];
+    const assetIdsToMarkOnline = [];
+    const pagination = usePagination(5000, (pagination) =>
+      this.assetRepository.getLibraryAssetPaths(pagination, library.id),
+    );
+
+    for await (const page of pagination) {
+      for (const asset of page) {
+        const isOffline = !crawledAssetPaths.has(asset.originalPath);
+        if (isOffline && !asset.isOffline) {
+          assetIdsToMarkOffline.push(asset.id);
+        }
 
-        this.logger.debug(`Will import ${filteredPaths.length} new asset(s)`);
+        if (!isOffline && asset.isOffline) {
+          assetIdsToMarkOnline.push(asset.id);
+        }
+
+        crawledAssetPaths.delete(asset.originalPath);
       }
+    }
+
+    if (assetIdsToMarkOffline.length > 0) {
+      this.logger.debug(`Found ${assetIdsToMarkOffline.length} offline asset(s) previously marked as online`);
+      await this.assetRepository.updateAll(assetIdsToMarkOffline, { isOffline: true });
+    }
+
+    if (assetIdsToMarkOnline.length > 0) {
+      this.logger.debug(`Found ${assetIdsToMarkOnline.length} online asset(s) previously marked as offline`);
+      await this.assetRepository.updateAll(assetIdsToMarkOnline, { isOffline: false });
+    }
+
+    if (!shouldScanAll) {
+      pathsToScan = [...crawledAssetPaths];
+      this.logger.debug(`Will import ${pathsToScan.length} new asset(s)`);
+    }
 
-      await this.scanAssets(job.id, filteredPaths, library.ownerId, job.refreshAllFiles ?? false);
+    if (pathsToScan.length > 0) {
+      await this.scanAssets(job.id, pathsToScan, library.ownerId, job.refreshAllFiles ?? false);
     }
 
     await this.repository.update({ id: job.id, refreshedAt: new Date() });

diff --git a/server/src/domain/repositories/asset.repository.ts b/server/src/domain/repositories/asset.repository.ts
@@ -109,6 +109,8 @@ export interface MetadataSearchOptions {
   numResults: number;
 }
 
+export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
+
 export const IAssetRepository = 'IAssetRepository';
 
 export interface IAssetRepository {
@@ -129,10 +131,8 @@ export interface IAssetRepository {
   getRandom(userId: string, count: number): Promise<AssetEntity[]>;
   getFirstAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
   getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
-  getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]>;
+  getLibraryAssetPaths(pagination: PaginationOptions, libraryId: string): Paginated<AssetPathEntity>;
   getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
-  getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]>;
-  updateOfflineLibraryAssets(libraryId: string, originalPaths: string[]): Promise<void>;
   deleteAll(ownerId: string): Promise<void>;
   getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
   getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;

diff --git a/server/src/infra/repositories/asset.repository.ts b/server/src/infra/repositories/asset.repository.ts
@@ -2,6 +2,7 @@ import {
   AssetBuilderOptions,
   AssetCreate,
   AssetExploreFieldOptions,
+  AssetPathEntity,
   AssetSearchOptions,
   AssetStats,
   AssetStatsOptions,
@@ -184,10 +185,10 @@ export class AssetRepository implements IAssetRepository {
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
-  @ChunkedArray()
-  getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]> {
-    return this.repository.find({
-      where: { library: { id: In(libraryIds) } },
+  getLibraryAssetPaths(pagination: PaginationOptions, libraryId: string): Paginated<AssetPathEntity> {
+    return paginate(this.repository, pagination, {
+      select: { id: true, originalPath: true, isOffline: true },
+      where: { library: { id: libraryId } },
     });
   }