Improve task creation with cloud storage and share data (cvat-ai#6074)

### Motivation and context Resolved cvat-ai#6037 Related cvat-ai#4400 Related cvat-ai#6028 ![image](https://user-images.githubusercontent.com/49038720/236890662-c44b578e-5808-4fde-a216-2dcab6e95ab0.png) Co-authored-by: Boris Sekachev <[email protected]> Co-authored-by: Boris Sekachev <[email protected]> Co-authored-by: Roman Donchenko <[email protected]> Co-authored-by: Nikita Manovich <[email protected]>
retailnext · May 20, 2023 · df31dc7 · df31dc7
1 parent 692b16c
commit df31dc7
Show file tree

Hide file tree

Showing 41 changed files with 1,932 additions and 1,078 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,13 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## \[2.5.0] - Unreleased
 ### Added
-- TDB
+- Support share point/cloud storage files and directories that should be excluded when creating a task (server) (<https://github.com/opencv/cvat/pull/6074>)
+- Support task creation with cloud storage/share directories (<https://github.com/opencv/cvat/pull/6074>)
+- Support task creation with any type of data supported by the server by default from cloud storage
+without use_cache option (<https://github.com/opencv/cvat/pull/6074>)
+- Support task creation with cloud storage data and without use_cache option (<https://github.com/opencv/cvat/pull/6074>)
+
+### Changed
+- Cloud storage manifest file is optional (<https://github.com/opencv/cvat/pull/6074>)
 
 ### Changed
 - TDB
 
 ### Deprecated
-- TDB
+- The endpoint /cloudstorages/{id}/content was deprecated (<https://github.com/opencv/cvat/pull/6074>)
 
 ### Removed
 - TDB

diff --git a/cvat-core/src/api-implementation.ts b/cvat-core/src/api-implementation.ts
@@ -3,6 +3,7 @@
 //
 // SPDX-License-Identifier: MIT
 
+import { omit } from 'lodash';
 import config from './config';
 
 import PluginRegistry from './plugins';
@@ -44,7 +45,7 @@ export default function implementAPI(cvat) {
 
     cvat.server.share.implementation = async (directory) => {
         const result = await serverProxy.server.share(directory);
-        return result;
+        return result.map((item) => ({ ...omit(item, 'mime_type'), mimeType: item.mime_type }));
     };
 
     cvat.server.formats.implementation = async () => {

diff --git a/cvat-core/src/cloud-storage.ts b/cvat-core/src/cloud-storage.ts
@@ -3,12 +3,14 @@
 //
 // SPDX-License-Identifier: MIT
 
+import { omit } from 'lodash';
 import PluginRegistry from './plugins';
 import serverProxy from './server-proxy';
 import { ArgumentError } from './exceptions';
 import { CloudStorageCredentialsType, CloudStorageProviderType, CloudStorageStatus } from './enums';
 import User from './user';
 import { decodePreview } from './frames';
+import { SerializedRemoteFile } from './server-response-types';
 
 function validateNotEmptyString(value: string): void {
     if (typeof value !== 'string') {
@@ -164,7 +166,6 @@ export default class CloudStorage {
                 manifestPath: {
                     get: () => data.manifest_path,
                     set: (value) => {
-                        validateNotEmptyString(value);
                         data.manifest_path = value;
                     },
                 },
@@ -245,8 +246,11 @@ export default class CloudStorage {
         return result;
     }
 
-    public async getContent(): Promise<any> {
-        const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getContent);
+    public async getContent(path: string, nextToken?: string): Promise<{
+        next: string | null,
+        content: (Omit<SerializedRemoteFile, 'mime_type'> & { mimeType: string })[],
+    }> {
+        const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getContent, path, nextToken);
         return result;
     }
 
@@ -361,9 +365,12 @@ Object.defineProperties(CloudStorage.prototype.getContent, {
     implementation: {
         writable: false,
         enumerable: false,
-        value: async function implementation(): Promise<any> {
-            const result = await serverProxy.cloudStorages.getContent(this.id, this.manifestPath);
-            return result;
+        value: async function implementation(path: string, nextToken?: string): ReturnType<CloudStorage['getContent']> {
+            const result = await serverProxy.cloudStorages.getContent(this.id, path, nextToken, this.manifestPath);
+            return {
+                next: result.next,
+                content: result.content.map((item) => ({ ...omit(item, 'mime_type'), mimeType: item.mime_type })),
+            };
         },
     },
 });
@@ -376,7 +383,7 @@ Object.defineProperties(CloudStorage.prototype.getPreview, {
             return new Promise((resolve, reject) => {
                 serverProxy.cloudStorages
                     .getPreview(this.id)
-                    .then((result) => decodePreview(result))
+                    .then((result) => ((result) ? decodePreview(result) : Promise.resolve(result)))
                     .then((decoded) => resolve(decoded))
                     .catch((error) => {
                         reject(error);

diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts
@@ -10,7 +10,7 @@ import * as tus from 'tus-js-client';
 import {
     SerializedLabel, SerializedAnnotationFormats, ProjectsFilter,
     SerializedProject, SerializedTask, TasksFilter, SerializedUser,
-    SerializedAbout, SerializedShare, SerializedUserAgreement,
+    SerializedAbout, SerializedRemoteFile, SerializedUserAgreement,
     SerializedRegister, JobsFilter, SerializedJob,
 } from 'server-response-types';
 import { Storage } from './storage';
@@ -309,7 +309,7 @@ async function about(): Promise<SerializedAbout> {
     return response.data;
 }
 
-async function share(directoryArg: string): Promise<SerializedShare[]> {
+async function share(directoryArg: string): Promise<SerializedRemoteFile[]> {
     const { backendAPI } = config;
 
     let response = null;
@@ -1368,7 +1368,7 @@ async function getUsers(filter = { page_size: 'all' }) {
     return response.data.results;
 }
 
-function getPreview(instance) {
+function getPreview(instance: 'projects' | 'tasks' | 'jobs' | 'cloudstorages') {
     return async function (id: number) {
         const { backendAPI } = config;
 
@@ -1383,7 +1383,7 @@ function getPreview(instance) {
             throw new ServerError(`Could not get preview for "${instance}/${id}"`, code);
         }
 
-        return response.data;
+        return (response.status === 200) ? response.data : '';
     };
 }
 
@@ -1860,14 +1860,17 @@ async function getCloudStorages(filter = {}) {
     return response.data.results;
 }
 
-async function getCloudStorageContent(id, manifestPath) {
+async function getCloudStorageContent(id: number, path: string, nextToken?: string, manifestPath?: string):
+Promise<{ content: SerializedRemoteFile[], next: string | null }> {
     const { backendAPI } = config;
 
     let response = null;
     try {
-        const url = `${backendAPI}/cloudstorages/${id}/content`;
+        const url = `${backendAPI}/cloudstorages/${id}/content-v2`;
         response = await Axios.get(url, {
             params: {
+                prefix: path,
+                ...(nextToken ? { next_token: nextToken } : {}),
                 ...(manifestPath ? { manifest_path: manifestPath } : {}),
             },
         });

diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts
@@ -152,7 +152,7 @@ export interface SerializedAbout {
     version: string;
 }
 
-export interface SerializedShare {
+export interface SerializedRemoteFile {
     name: string;
     type: ShareFileType;
     mime_type: string;

diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py
@@ -96,6 +96,7 @@ def upload_data(
                     "job_file_mapping",
                     "filename_pattern",
                     "cloud_storage_id",
+                    "server_files_exclude",
                 ],
             )
         )

diff --git a/cvat-ui/src/actions/share-actions.ts b/cvat-ui/src/actions/share-actions.ts
diff --git a/cvat-ui/src/components/cloud-storages-page/cloud-storages-filter-configuration.ts b/cvat-ui/src/components/cloud-storages-page/cloud-storages-filter-configuration.ts
@@ -1,4 +1,5 @@
 // Copyright (C) 2022 Intel Corporation
+// Copyright (C) 2023 CVAT.ai Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -46,8 +47,8 @@ export const config: Partial<Config> = {
             valueSources: ['value'],
             operators: ['like'],
         },
-        display_name: {
-            label: 'Display name',
+        name: {
+            label: 'Name',
             type: 'text',
             valueSources: ['value'],
             operators: ['like'],

diff --git a/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx b/cvat-ui/src/components/create-cloud-storage-page/manifests-manager.tsx
@@ -7,12 +7,13 @@ import React, { useEffect } from 'react';
 import { DeleteOutlined, PlusCircleOutlined, QuestionCircleOutlined } from '@ant-design/icons';
 import Button from 'antd/lib/button';
 import Col from 'antd/lib/col';
-import Form, { RuleObject } from 'antd/lib/form';
+import Form from 'antd/lib/form';
 import { FormListFieldData, FormListOperation } from 'antd/lib/form/FormList';
 import Input from 'antd/lib/input';
 import Row from 'antd/lib/row';
 import Tooltip from 'antd/lib/tooltip';
 import config from 'config';
+import { Alert } from 'antd';
 
 interface Props {
     form: any;
@@ -71,19 +72,9 @@ export default function ManifestsManager(props: Props): JSX.Element {
                         </Tooltip>
                     </>
                 )}
-                required
             />
             <Form.List
                 name='manifests'
-                rules={[
-                    {
-                        validator: async (_: RuleObject, names: string[]): Promise<void> => {
-                            if (!names || !names.length) {
-                                throw new Error('Please, specify at least one manifest file');
-                            }
-                        },
-                    },
-                ]}
             >
                 {
                     (fields: FormListFieldData[], _: FormListOperation, { errors }: { errors: React.ReactNode[] }) => (
@@ -135,6 +126,13 @@ export default function ManifestsManager(props: Props): JSX.Element {
                     </Button>
                 </Col>
             </Row>
+            {!manifestNames.length && (
+                <Row>
+                    <Col>
+                        <Alert type='info' message='We highly recommend attaching a manifest file to reduce the number of requests to the bucket.' />
+                    </Col>
+                </Row>
+            )}
         </>
     );
 }