Feature - Add upload functionality on Web (immich-app#231)

* Added file selector

* Extract metadata to upload files to the web

* Added request for uploading

* Generate jpeg/Webp thumbnail for asset uploaded without thumbnail data

* Added generating thumbnail for video and WebSocket broadcast after thumbnail is generated

* Added video length extraction

* Added Uploading Panel

* Added upload progress store and styling the uploaded asset

* Added condition to only show upload panel when there is upload in progress

* Remove asset from the upload list after successfully uploading

* Added WebSocket to listen to upload event on the web

* Added mechanism to check for existing assets before uploading on the web

* Added test workflow

* Update readme

.github/workflows/test.yml

@@ -0,0 +1,17 @@
+name: Test
+on:
+  pull_request:
+  push: { branches: master }
+
+jobs:
+  test-server-e2e:
+    name: Run test suite
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Run Immich Server 2E2 Test
+        run: docker-compose -f ./docker/docker-compose.test.yml --env-file ./docker/.env.test up --abort-on-container-exit --exit-code-from immich_server_test

README.md

@@ -8,7 +8,7 @@
     <img src="https://img.shields.io/teamcity/http/immichci.little-home.net/s/Immich_BuildAndPublishIOSToTestFlight.svg?style=for-the-badge&label=iOS&logo=teamcity&logoColor=000000&labelColor=ececec" alt="iOS Build"/>
   </a>
   <a href="https://actions-badge.atrox.dev/alextran1502/immich/goto?ref=main">
-    <img alt="Build Status" src="https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Falextran1502%2Fimmich%2Fbadge%3Fref%3Dmain&style=for-the-badge&label=Server Docker&logo=docker&labelColor=ececec" />
+    <img alt="Build Status" src="https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Falextran1502%2Fimmich%2Fbadge%3Fref%3Dmain&style=for-the-badge&label=Github Action&logo=github&labelColor=ececec&logoColor=000000" />
   </a>
   <a href="https://discord.gg/rxnyVTXGbM">
     <img src="https://img.shields.io/discord/979116623879368755.svg?label=Immich%20Discord&logo=Discord&style=for-the-badge&logoColor=000000&labelColor=ececec" atl="Immich Discord"/>
@@ -25,15 +25,15 @@
 
 # Immich
 
-Self-hosted photo and video backup solution directly from your mobile phone.
+**High performance self-hosted photo and video backup solution.**
 
 ![](https://media.giphy.com/media/y8ZeaAigGmNvlSoKhU/giphy.gif)
 
 Loading ~4000 images/videos
 
 ## Screenshots
 
-### Mobile client
+### Mobile
 <p align="left">
   <img src="design/login-screen.png" width="150" title="Login With Custom URL">
   <img src="design/backup-screen.png" width="150" title="Backup Setting Info">
@@ -44,9 +44,10 @@ Loading ~4000 images/videos
   <img src="design/nsc6.png" width="150" title="EXIF Info">
 </p>
 
-### Web client
-<p align="center">
-  <img src="design/dashboard_photos.jpeg" width="100%" title="Home Dashboard">
+### Web
+<p align="left">
+  <img src="design/web-home.jpeg"  width="49%" title="Home Dashboard">
+  <img src="design/web-detail.jpeg" width="49%" title="Detail">
 </p>
 
 # Note
@@ -55,26 +56,22 @@ Loading ~4000 images/videos
 
 This project is under heavy development, there will be continuous functions, features and api changes.
 
-# Features
-
-- Upload and view assets (videos/images).
-- Auto Backup.
-- Download asset to local device.
-- Multi-user supported.
-- Quick navigation with drag scroll bar.
-- Support HEIC/HEIF Backup.
-- Extract and display EXIF info.
-- Real-time render from multi-device upload event.
-- Image Tagging/Classification based on ImageNet dataset
-- Object detection based on COCO SSD.
-- Search assets based on tags and exif data (lens, make, model, orientation)
-- [Optional] Reverse geocoding using Mapbox (Generous free-tier of 100,000 search/month)
-- Show asset's location information on map (OpenStreetMap).
-- Show curated places on the search page
-- Show curated objects on the search page
-- Shared album with users on the same server
-- Selective backup - albums can be included and excluded during the backup process.
-- Web interface is available for administrative tasks (creating new users) and viewing assets on the server - additional features are coming.
+# Features 
+
+|  | Mobile | Web |
+| - | - | - |
+| Upload and view videos and photos | Yes | Yes
+| Auto backup when app is opened | Yes | N/A
+| Selective album(s) for backup | Yes | N/A
+| Download photos and videos to local device | Yes | Yes
+| Multi-user support | Yes | Yes
+| Shared Albums | Yes | No
+| Quick navigation with draggable scrollbar | Yes | Yes
+| Support RAW (HEIC, HEIF, DNG, Apple ProRaw) | Yes | Yes
+| Metadata view (EXIF, map) | Yes | Yes
+| Search by metadata, objects and image tags | Yes | No
+| Administrative functions (user management) | No | Yes
+
 
 # System Requirement
 
@@ -97,7 +94,7 @@ You can use docker compose for development and testing out the application, ther
 3. **PostgreSQL** - Main database of the application
 4. **Redis** - For sharing websocket instance between docker instances and background tasks message queue.
 5. **Nginx** - Load balancing and optimized file uploading.
-6. **TensorFlow** - Object Detection and Image Classification.
+6. **TensorFlow** - Object Detection (COCO SSD) and Image Classification (ImageNet).
 
 ## Step 1: Populate .env file
 

docker/docker-compose.dev.yml

@@ -60,6 +60,7 @@ services:
       - NODE_ENV=development
     depends_on:
       - database
+      - immich-server
     networks:
       - immich-network
 

server/apps/immich/src/api-v1/asset/asset.controller.ts

@@ -15,6 +15,7 @@ import {
   Delete,
   Logger,
   Patch,
+  HttpCode,
 } from '@nestjs/common';
 import { JwtAuthGuard } from '../../modules/immich-jwt/guards/jwt-auth.guard';
 import { AssetService } from './asset.service';
@@ -76,15 +77,17 @@ export class AssetController {
             { asset: assetWithThumbnail, fileName: file.originalname, fileSize: file.size, hasThumbnail: true },
             { jobId: savedAsset.id },
           );
+
+          this.wsCommunicateionGateway.server
+            .to(savedAsset.userId)
+            .emit('on_upload_success', JSON.stringify(assetWithThumbnail));
         } else {
           await this.assetUploadedQueue.add(
             'asset-uploaded',
             { asset: savedAsset, fileName: file.originalname, fileSize: file.size, hasThumbnail: false },
             { jobId: savedAsset.id },
           );
         }
-
-        this.wsCommunicateionGateway.server.to(savedAsset.userId).emit('on_upload_success', JSON.stringify(savedAsset));
       } catch (e) {
         Logger.error(`Error receiving upload file ${e}`);
       }
@@ -171,4 +174,20 @@ export class AssetController {
 
     return result;
   }
+
+  /**
+   * Check duplicated asset before uploading - for Web upload used
+   */
+  @Post('/check')
+  @HttpCode(200)
+  async checkDuplicateAsset(
+    @GetAuthUser() authUser: AuthUserDto,
+    @Body(ValidationPipe) { deviceAssetId }: { deviceAssetId: string },
+  ) {
+    const res = await this.assetService.checkDuplicatedAsset(authUser, deviceAssetId);
+
+    return {
+      isExist: res,
+    };
+  }
 }

server/apps/immich/src/api-v1/asset/asset.module.ts

@@ -24,6 +24,6 @@ import { CommunicationModule } from '../communication/communication.module';
   ],
   controllers: [AssetController],
   providers: [AssetService, BackgroundTaskService],
-  exports: [],
+  exports: [AssetService],
 })
 export class AssetModule {}

server/apps/immich/src/api-v1/asset/asset.service.ts

@@ -1,6 +1,6 @@
 import { BadRequestException, Injectable, InternalServerErrorException, Logger, StreamableFile } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
-import { Repository } from 'typeorm';
+import { IsNull, Not, Repository } from 'typeorm';
 import { AuthUserDto } from '../../decorators/auth-user.decorator';
 import { CreateAssetDto } from './dto/create-asset.dto';
 import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
@@ -72,6 +72,7 @@ export class AssetService {
       return await this.assetRepository.find({
         where: {
           userId: authUser.id,
+          resizePath: Not(IsNull()),
         },
         relations: ['exifInfo'],
         order: {
@@ -381,4 +382,15 @@ export class AssetService {
       [authUser.id],
     );
   }
+
+  async checkDuplicatedAsset(authUser: AuthUserDto, deviceAssetId: string) {
+    const res = await this.assetRepository.findOne({
+      where: {
+        deviceAssetId,
+        userId: authUser.id,
+      },
+    });
+
+    return res ? true : false;
+  }
 }

server/apps/immich/src/api-v1/communication/communication.gateway.ts

@@ -6,8 +6,9 @@ import { Logger } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 import { UserEntity } from '@app/database/entities/user.entity';
 import { Repository } from 'typeorm';
+import { query } from 'express';
 
-@WebSocketGateway()
+@WebSocketGateway({ cors: true })
 export class CommunicationGateway implements OnGatewayConnection, OnGatewayDisconnect {
   constructor(
     private immichJwtService: ImmichJwtService,
@@ -21,27 +22,33 @@ export class CommunicationGateway implements OnGatewayConnection, OnGatewayDisco
   handleDisconnect(client: Socket) {
     client.leave(client.nsp.name);
 
-    Logger.log(`Client ${client.id} disconnected`);
+    Logger.log(`Client ${client.id} disconnected from Websocket`, 'WebsocketConnectionEvent');
   }
 
   async handleConnection(client: Socket, ...args: any[]) {
-    Logger.log(`New websocket connection: ${client.id}`, 'NewWebSocketConnection');
-    const accessToken = client.handshake.headers.authorization.split(' ')[1];
-    const res = await this.immichJwtService.validateToken(accessToken);
-
-    if (!res.status) {
-      client.emit('error', 'unauthorized');
-      client.disconnect();
-      return;
-    }
+    try {
+      Logger.log(`New websocket connection: ${client.id}`, 'WebsocketConnectionEvent');
 
-    const user = await this.userRepository.findOne({ where: { id: res.userId } });
-    if (!user) {
-      client.emit('error', 'unauthorized');
-      client.disconnect();
-      return;
-    }
+      const accessToken = client.handshake.headers.authorization.split(' ')[1];
+
+      const res = await this.immichJwtService.validateToken(accessToken);
 
-    client.join(user.id);
+      if (!res.status) {
+        client.emit('error', 'unauthorized');
+        client.disconnect();
+        return;
+      }
+
+      const user = await this.userRepository.findOne({ where: { id: res.userId } });
+      if (!user) {
+        client.emit('error', 'unauthorized');
+        client.disconnect();
+        return;
+      }
+
+      client.join(user.id);
+    } catch (e) {
+      // Logger.error(`Error establish websocket conneciton ${e}`, 'HandleWebscoketConnection');
+    }
   }
 }

server/apps/microservices/src/main.ts

@@ -1,10 +1,13 @@
 import { Logger } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
+import { RedisIoAdapter } from '../../immich/src/middlewares/redis-io.adapter.middleware';
 import { MicroservicesModule } from './microservices.module';
 
 async function bootstrap() {
   const app = await NestFactory.create(MicroservicesModule);
 
+  app.useWebSocketAdapter(new RedisIoAdapter(app));
+
   await app.listen(3000, () => {
     if (process.env.NODE_ENV == 'development') {
       Logger.log('Running Immich Microservices in DEVELOPMENT environment', 'ImmichMicroservice');

server/apps/microservices/src/microservices.module.ts

@@ -11,6 +11,9 @@ import { AssetUploadedProcessor } from './processors/asset-uploaded.processor';
 import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor';
 import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
 import { VideoTranscodeProcessor } from './processors/video-transcode.processor';
+import { AssetModule } from '../../immich/src/api-v1/asset/asset.module';
+import { CommunicationGateway } from '../../immich/src/api-v1/communication/communication.gateway';
+import { CommunicationModule } from '../../immich/src/api-v1/communication/communication.module';
 
 @Module({
   imports: [
@@ -56,6 +59,7 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
         removeOnFail: false,
       },
     }),
+    CommunicationModule,
   ],
   controllers: [],
   providers: [

server/apps/microservices/src/processors/asset-uploaded.processor.ts

@@ -46,6 +46,7 @@ export class AssetUploadedProcessor {
       await this.metadataExtractionQueue.add('detect-object', { asset }, { jobId: randomUUID() });
     } else {
       // Generate Thumbnail -> Then generate webp, tag image and detect object
+      await this.thumbnailGeneratorQueue.add('generate-jpeg-thumbnail', { asset }, { jobId: randomUUID() });
     }
 
     // Video Conversion
@@ -63,5 +64,10 @@ export class AssetUploadedProcessor {
         { jobId: randomUUID() },
       );
     }
+
+    // Extract video duration if uploaded from the web
+    if (asset.type == AssetType.VIDEO && asset.duration == '0:00:00.000000') {
+      await this.metadataExtractionQueue.add('extract-video-length', { asset }, { jobId: randomUUID() });
+    }
   }
 }

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -12,6 +12,8 @@ import { Logger } from '@nestjs/common';
 import axios from 'axios';
 import { SmartInfoEntity } from '@app/database/entities/smart-info.entity';
 import { ConfigService } from '@nestjs/config';
+import ffmpeg from 'fluent-ffmpeg';
+// import moment from 'moment';
 
 @Processor('metadata-extraction-queue')
 export class MetadataExtractionProcessor {
@@ -129,4 +131,27 @@ export class MetadataExtractionProcessor {
       Logger.error(`Failed to trigger object detection pipe line ${error.toString()}`);
     }
   }
+
+  @Process({ name: 'extract-video-length', concurrency: 2 })
+  async extractVideoLength(job: Job) {
+    const { asset }: { asset: AssetEntity } = job.data;
+
+    ffmpeg.ffprobe(asset.originalPath, async (err, data) => {
+      if (!err) {
+        if (data.format.duration) {
+          const videoDurationInSecond = parseInt(data.format.duration.toString(), 0);
+
+          const hours = Math.floor(videoDurationInSecond / 3600);
+          const minutes = Math.floor((videoDurationInSecond - hours * 3600) / 60);
+          const seconds = videoDurationInSecond - hours * 3600 - minutes * 60;
+
+          const durationString = `${hours}:${minutes < 10 ? '0' + minutes.toString() : minutes}:${
+            seconds < 10 ? '0' + seconds.toString() : seconds
+          }.000000`;
+
+          await this.assetRepository.update({ id: asset.id }, { duration: durationString });
+        }
+      }
+    });
+  }
 }