PS-579 uploader - allowed types (#364)

.env

@@ -89,6 +89,7 @@ UPLOADER_STORAGE_USE_PATH_STYLE_ENDPOINT=true
 UPLOADER_REQUEST_SIGNATURE_TTL=600
 UPLOADER_DELETE_ASSET_GRACEFUL_TIME=30
 UPLOADER_RABBITMQ_VHOST=uploader
+UPLOADER_ALLOWED_FILE_TYPES='image/*(jpg,jpeg,bmp,tif,gif,png,heic),application/*(pdf,doc,docx,xls,xlsx,odt),video/*(mpg,mpeg,mov,avi,mp3,mp2,mp4,m4v,m4a,mkv,hevc)audio/*(aac,aiff,wav)'
 
 # For admin OAuth clients
 AUTH_ADMIN_CLIENT_ID=auth-admin

databox/api/src/Api/DataTransformer/AbstractFileInputDataTransformer.php

@@ -5,14 +5,14 @@
 namespace App\Api\DataTransformer;
 
 use Alchemy\StorageBundle\Upload\UploadManager;
+use Alchemy\StorageBundle\Util\FileUtil;
 use App\Api\Model\Input\AssetSourceInput;
 use App\Consumer\Handler\File\ImportFileHandler;
 use App\Doctrine\Listener\PostFlushStack;
 use App\Entity\Core\File;
 use App\Entity\Core\Workspace;
 use App\Http\FileUploadManager;
 use App\Storage\RenditionManager;
-use App\Util\FileUtil;
 use Symfony\Component\HttpFoundation\File\UploadedFile;
 use Symfony\Component\HttpFoundation\RequestStack;
 use Symfony\Component\HttpKernel\Exception\BadRequestHttpException;

databox/api/src/Asset/FileCopier.php

@@ -5,10 +5,10 @@
 namespace App\Asset;
 
 use Alchemy\StorageBundle\Storage\FileStorageManager;
+use Alchemy\StorageBundle\Util\FileUtil;
 use App\Entity\Core\File;
 use App\Entity\Core\Workspace;
 use App\Storage\FilePathGenerator;
-use App\Util\FileUtil;
 use Doctrine\ORM\EntityManagerInterface;
 
 class FileCopier

databox/api/src/Controller/Core/ExportAction.php

@@ -12,7 +12,7 @@
 use App\Model\Export;
 use App\Repository\Core\AssetRenditionRepository;
 use App\Security\RenditionPermissionManager;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 use Doctrine\ORM\EntityManagerInterface;
 use GuzzleHttp\Client;
 use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;

databox/api/src/Integration/Aws/Rekognition/AbstractRekognitionAction.php

@@ -4,13 +4,13 @@
 
 namespace App\Integration\Aws\Rekognition;
 
+use Alchemy\StorageBundle\Util\FileUtil;
 use Alchemy\Workflow\Executor\RunContext;
 use App\Attribute\BatchAttributeManager;
 use App\Entity\Core\Asset;
 use App\Integration\AbstractIntegrationAction;
 use App\Integration\IfActionInterface;
 use App\Storage\RenditionManager;
-use App\Util\FileUtil;
 
 abstract class AbstractRekognitionAction extends AbstractIntegrationAction implements IfActionInterface
 {

databox/api/src/Integration/Aws/Rekognition/AwsRekognitionIntegration.php

@@ -11,7 +11,7 @@
 use App\Integration\FileActionsIntegrationInterface;
 use App\Integration\WorkflowHelper;
 use App\Integration\WorkflowIntegrationInterface;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 use Symfony\Component\Config\Definition\Builder\NodeBuilder;
 use Symfony\Component\Config\Definition\Builder\NodeDefinition;
 use Symfony\Component\Config\Definition\Builder\TreeBuilder;

databox/api/src/Integration/Aws/Transcribe/AwsTranscribeIntegration.php

@@ -13,7 +13,7 @@
 use App\Integration\Aws\AbstractAwsIntegration;
 use App\Integration\WorkflowHelper;
 use App\Integration\WorkflowIntegrationInterface;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 use App\Util\LocaleUtils;
 use Symfony\Component\Config\Definition\Builder\NodeBuilder;
 

databox/api/src/Integration/Aws/Transcribe/TranscribeAction.php

@@ -10,7 +10,7 @@
 use App\Integration\ApiBudgetLimiter;
 use App\Integration\IfActionInterface;
 use App\Storage\S3Copier;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 
 final class TranscribeAction extends AbstractIntegrationAction implements IfActionInterface
 {

databox/api/src/Integration/Blurhash/BlurhashAction.php

@@ -16,7 +16,7 @@
 use App\Integration\AbstractIntegrationAction;
 use App\Integration\IfActionInterface;
 use App\Storage\RenditionManager;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 use kornrunner\Blurhash\Blurhash;
 
 class BlurhashAction extends AbstractIntegrationAction implements IfActionInterface

databox/api/src/Integration/Clarifai/ClarifaiConceptsAction.php

@@ -11,7 +11,7 @@
 use App\Entity\Core\Asset;
 use App\Integration\AbstractIntegrationAction;
 use App\Integration\IfActionInterface;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 
 class ClarifaiConceptsAction extends AbstractIntegrationAction implements IfActionInterface
 {

databox/api/src/Integration/RemoveBg/RemoveBgAction.php

@@ -7,7 +7,7 @@
 use Alchemy\Workflow\Executor\RunContext;
 use App\Entity\Core\Asset;
 use App\Integration\AbstractIntegrationAction;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 
 class RemoveBgAction extends AbstractIntegrationAction
 {

databox/api/src/Integration/RemoveBg/RemoveBgIntegration.php

@@ -9,7 +9,7 @@
 use App\Integration\AbstractFileAction;
 use App\Integration\WorkflowHelper;
 use App\Integration\WorkflowIntegrationInterface;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 use Symfony\Component\Config\Definition\Builder\NodeBuilder;
 use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\Request;

databox/api/src/Integration/ToastUi/TuiPhotoEditorIntegration.php

@@ -8,7 +8,7 @@
 use App\Entity\Integration\WorkspaceIntegration;
 use App\Integration\AbstractFileAction;
 use App\Integration\FileActionsIntegrationInterface;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 use Symfony\Component\HttpFoundation\JsonResponse;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;

databox/api/src/Storage/FileManager.php

@@ -7,7 +7,7 @@
 use Alchemy\StorageBundle\Storage\FileStorageManager;
 use App\Entity\Core\File;
 use App\Entity\Core\Workspace;
-use App\Util\FileUtil;
+use Alchemy\StorageBundle\Util\FileUtil;
 use Doctrine\ORM\EntityManagerInterface;
 
 class FileManager

databox/client/Dockerfile

@@ -20,7 +20,7 @@ RUN apk add --no-cache libstdc++ \
     && apk add --virtual .build \
         wget \
     && mkdir -p /var/docker \
-    && wget -q -O /var/docker/generate-env https://github.com/alchemy-fr/config-compiler/releases/download/v2.0.0/generate-env-alpine \
+    && wget -q -O /var/docker/generate-env https://github.com/alchemy-fr/config-compiler/releases/download/v2.2.1/generate-env-alpine \
     && chmod +x /var/docker/generate-env \
     && apk del .build \
     && rm /etc/nginx/conf.d/default.conf

databox/client/config-compiler.js

@@ -51,6 +51,25 @@
         }
     }
 
+    const normalizeTypes = (value) => {
+        if (!value) {
+            return {};
+        }
+        const v = value.trim();
+
+        if (!v) {
+            return {};
+        }
+
+        const types = [...v.matchAll(/([\w*]+\/[\w*+.-]+)(\([\w,]*\))?/g)];
+        const struct = {};
+        for (const t of types) {
+            struct[t[1]] = t[2] ? t[2].substring(1, t[2].length - 1).split(',').map(e => e.trim()).filter(e => !!e) : [];
+        }
+
+        return struct;
+    };
+
     return {
         customHTML: {
             __TPL_HEAD__: scriptTpl,
@@ -68,5 +87,6 @@
         requestSignatureTtl: env.S3_REQUEST_SIGNATURE_TTL,
         displayServicesMenu: env.DISPLAY_SERVICES_MENU === 'true',
         dashboardBaseUrl: env.DASHBOARD_BASE_URL,
+        allowedTypes: normalizeTypes(env.ALLOWED_FILE_TYPES),
     };
 });

databox/client/src/components/Media/Asset/AssetDropzone.tsx

@@ -5,6 +5,7 @@ import UploadModal from "../../Upload/UploadModal";
 import {Backdrop, Typography} from "@mui/material";
 import {retrieveImageFromClipboardAsBlob} from "../../../lib/ImagePaste";
 import {useModals} from "../../../hooks/useModalStack";
+import {useAccept} from "../../Upload/UploadDropzone";
 
 export default function AssetDropzone({children}: PropsWithChildren<{}>) {
     const userContext = useContext(UserContext);
@@ -40,10 +41,13 @@ export default function AssetDropzone({children}: PropsWithChildren<{}>) {
         }
     }, []);
 
+    const accept = useAccept();
+
     const {getRootProps, getInputProps, isDragActive} = useDropzone({
         noClick: true,
         onDrop,
         noKeyboard: true,
+        accept,
     });
 
     return <div {...getRootProps()}>

databox/client/src/components/Upload/UploadDropzone.tsx

@@ -1,18 +1,32 @@
 import React from 'react';
-import Dropzone, {DropzoneOptions} from "react-dropzone";
+import Dropzone, {Accept, DropzoneOptions} from "react-dropzone";
 import {Box, Typography} from "@mui/material";
 import {grey} from "@mui/material/colors";
+import config from "../../config";
+
+export function useAccept() {
+    return React.useMemo<Accept | undefined>(() => {
+        const list = [
+            ...(config.get('allowedTypes') as string[]),
+            ...(config.get('allowedExtensions') as string[]).map(e => `.${e}`),
+        ];
+
+        return list.length > 0 ? {'image/*': list} : undefined;
+    }, []);
+}
 
 type Props = {
     onDrop: DropzoneOptions['onDrop'];
 };
 
 export default function UploadDropzone({
-    onDrop
+    onDrop,
 }: Props) {
+    const accept = useAccept();
 
     return <Dropzone
         onDrop={onDrop}
+        accept={accept}
     >
         {({getRootProps, getInputProps, isDragActive}) => (
             <Box

docker-compose.dev.yml

@@ -175,6 +175,7 @@ services:
       - CLARIFAI_API_KEY
       - SECRETS_PUBLIC_KEY=${DATABOX_SECRETS_PUBLIC_KEY}
       - SECRETS_SECRET_KEY=${DATABOX_SECRETS_SECRET_KEY}
+      - ALLOWED_FILE_TYPES=${UPLOADER_ALLOWED_FILE_TYPES}
     working_dir: /var/workspace
     volumes:
       - ./:/var/workspace

docker-compose.yml

@@ -120,6 +120,7 @@ services:
       - NEWRELIC_ENABLED
       - NEWRELIC_LICENSE_KEY
       - NEWRELIC_APP_NAME=uploader-api
+      - ALLOWED_FILE_TYPES=${UPLOADER_ALLOWED_FILE_TYPES}
     volumes:
       - uploader_vol:/var/data/upload
       - ./configs:/configs
@@ -168,6 +169,7 @@ services:
       - NEWRELIC_ENABLED
       - NEWRELIC_LICENSE_KEY
       - NEWRELIC_APP_NAME=uploader-worker
+      - ALLOWED_FILE_TYPES=${UPLOADER_ALLOWED_FILE_TYPES}
     volumes:
       - uploader_vol:/var/data/upload
       - ./configs:/configs
@@ -221,6 +223,7 @@ services:
       - DISPLAY_SERVICES_MENU
       - DASHBOARD_BASE_URL=https://dashboard.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}
       - AUTO_CONNECT_IDP
+      - ALLOWED_FILE_TYPES=${UPLOADER_ALLOWED_FILE_TYPES}
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.uploader-client-dev.rule=Host(`uploader.${PHRASEA_DOMAIN}`)"
@@ -281,6 +284,7 @@ services:
       - DASHBOARD_BASE_URL=https://dashboard.${PHRASEA_DOMAIN}${HTTPS_PORT_PREFIX}
       - UPLOADER_TARGET_SLUG=${DATABOX_UPLOADER_TARGET_SLUG}
       - AUTO_CONNECT_IDP
+      - ALLOWED_FILE_TYPES=${UPLOADER_ALLOWED_FILE_TYPES}
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.databox-client-dev.rule=Host(`databox.${PHRASEA_DOMAIN}`)"

expose/client/Dockerfile

@@ -20,7 +20,7 @@ RUN apk add --no-cache libstdc++ \
     && apk add --virtual .build \
         wget \
     && mkdir -p /var/docker \
-    && wget -q -O /var/docker/generate-env https://github.com/alchemy-fr/config-compiler/releases/download/v2.0.0/generate-env-alpine \
+    && wget -q -O /var/docker/generate-env https://github.com/alchemy-fr/config-compiler/releases/download/v2.2.1/generate-env-alpine \
     && chmod +x /var/docker/generate-env \
     && apk del .build \
     && rm /etc/nginx/conf.d/default.conf

infra/docker/nodejs-base/Dockerfile

@@ -5,7 +5,7 @@ RUN apk add --no-cache \
         make \
         g++ \
     && mkdir -p /var/docker \
-    && wget -q -O /var/docker/generate-env --no-verbose https://github.com/alchemy-fr/config-compiler/releases/download/v2.0.0/generate-env-alpine \
+    && wget -q -O /var/docker/generate-env --no-verbose https://github.com/alchemy-fr/config-compiler/releases/download/v2.2.1/generate-env-alpine \
     && chmod +x /var/docker/generate-env \
     && mkdir -p /usr/src/app
 

lib/php/storage-bundle/DependencyInjection/AlchemyStorageExtension.php

@@ -26,6 +26,8 @@ public function load(array $configs, ContainerBuilder $container)
         $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config'));
         $loader->load('services.yaml');
         $loader->load('cdn.yaml');
+
+        $container->setParameter('alchemy_storage.upload.allowed_types', $config['upload']['allowed_types']);
     }
 
     public function prepend(ContainerBuilder $container)

lib/php/storage-bundle/DependencyInjection/Configuration.php

@@ -20,6 +20,16 @@ public function getConfigTreeBuilder()
         $treeBuilder = new TreeBuilder('alchemy_storage');
         $treeBuilder->getRootNode()
             ->children()
+                ->arrayNode('upload')
+                    ->addDefaultsIfNotSet()
+                    ->children()
+                        ->scalarNode('allowed_types')
+                            ->defaultValue('%env(ALLOWED_FILE_TYPES)%')
+                            ->info('comma separated values of MIME types, optionally including their extensions')
+                            ->example('image/*(jpg,jpeg,png,heic),application/*(pdf,doc,docx),video/*')
+                        ->end()
+                    ->end()
+                ->end()
             ->end()
         ;
 

lib/php/storage-bundle/Resources/config/services.yaml

@@ -6,6 +6,7 @@ parameters:
     env(S3_REQUEST_SIGNATURE_TTL): '3600'
     env(S3_PATH_PREFIX): ''
     env(VERIFY_SSL): true
+    env(ALLOWED_FILE_TYPES): ''
 
 services:
     _defaults:
@@ -56,3 +57,6 @@ services:
         $bucketName: '%env(S3_BUCKET_NAME)%'
         $ttl: '%env(int:S3_REQUEST_SIGNATURE_TTL)%'
         $pathPrefix: '%env(S3_PATH_PREFIX)%'
+
+    Alchemy\StorageBundle\Upload\FileValidator:
+        $allowedTypes: '%alchemy_storage.upload.allowed_types%'