Added SiamMask CUDA implementation (tracking), reworked tracking appr…

…oach (cvat-ai#3571)
kietvo · Sep 29, 2021 · dbdcd4f · dbdcd4f
1 parent 02172ad
commit dbdcd4f
Show file tree

Hide file tree

Showing 29 changed files with 675 additions and 293 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -46,13 +46,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support cloud storage status (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 - Support cloud storage preview (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 - cvat-core: support cloud storages (<https://github.com/openvinotoolkit/cvat/pull/3313>)
+- Added GPU implementation for SiamMask, reworked tracking approach (<https://github.com/openvinotoolkit/cvat/pull/3571>)
 
 ### Changed
 
 - Non-blocking UI when using interactors (<https://github.com/openvinotoolkit/cvat/pull/3473>)
 - "Selected opacity" slider now defines opacity level for shapes being drawnSelected opacity (<https://github.com/openvinotoolkit/cvat/pull/3473>)
 - Cloud storage creating and updating (<https://github.com/openvinotoolkit/cvat/pull/3386>)
 - Way of working with cloud storage content (<https://github.com/openvinotoolkit/cvat/pull/3386>)
+- UI tracking has been reworked (<https://github.com/openvinotoolkit/cvat/pull/3571>)
 
 ### Removed
 

diff --git a/cvat-canvas/package-lock.json b/cvat-canvas/package-lock.json
diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cvat-canvas",
-  "version": "2.7.0",
+  "version": "2.8.0",
   "description": "Part of Computer Vision Annotation Tool which presents its canvas library",
   "main": "src/canvas.ts",
   "scripts": {

diff --git a/cvat-canvas/src/scss/canvas.scss b/cvat-canvas/src/scss/canvas.scss
@@ -46,6 +46,12 @@ polyline.cvat_shape_drawing_opacity {
     pointer-events: none;
 }
 
+.cvat_canvas_text_description {
+    font-size: 14px;
+    fill: yellow;
+    font-style: oblique 40deg;
+}
+
 .cvat_canvas_crosshair {
     stroke: red;
 }

diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts
@@ -1458,6 +1458,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
             shapeType: state.shapeType,
             points: [...state.points],
             attributes: { ...state.attributes },
+            descriptions: [...state.descriptions],
             zOrder: state.zOrder,
             pinned: state.pinned,
             updated: state.updated,
@@ -1544,7 +1545,14 @@ export class CanvasViewImpl implements CanvasView, Listener {
                 }
             }
 
-            if (drawnState.label.id !== state.label.id) {
+            const stateDescriptions = state.descriptions;
+            const drawnStateDescriptions = drawnState.descriptions;
+
+            if (
+                drawnState.label.id !== state.label.id
+                || drawnStateDescriptions.length !== stateDescriptions.length
+                || drawnStateDescriptions.some((desc: string, id: number): boolean => desc !== stateDescriptions[id])
+            ) {
                 // need to remove created text and create it again
                 if (text) {
                     text.remove();
@@ -1967,7 +1975,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
     private addText(state: any): SVG.Text {
         const { undefinedAttrValue } = this.configuration;
         const {
-            label, clientID, attributes, source,
+            label, clientID, attributes, source, descriptions,
         } = state;
         const attrNames = label.attributes.reduce((acc: any, val: any): void => {
             acc[val.id] = val.name;
@@ -1977,13 +1985,25 @@ export class CanvasViewImpl implements CanvasView, Listener {
         return this.adoptedText
             .text((block): void => {
                 block.tspan(`${label.name} ${clientID} (${source})`).style('text-transform', 'uppercase');
+                for (const desc of descriptions) {
+                    block
+                        .tspan(`${desc}`)
+                        .attr({
+                            dy: '1em',
+                            x: 0,
+                        })
+                        .addClass('cvat_canvas_text_description');
+                }
                 for (const attrID of Object.keys(attributes)) {
                     const value = attributes[attrID] === undefinedAttrValue ? '' : attributes[attrID];
-                    block.tspan(`${attrNames[attrID]}: ${value}`).attr({
-                        attrID,
-                        dy: '1em',
-                        x: 0,
-                    });
+                    block
+                        .tspan(`${attrNames[attrID]}: ${value}`)
+                        .attr({
+                            attrID,
+                            dy: '1em',
+                            x: 0,
+                        })
+                        .addClass('cvat_canvas_text_attribute');
                 }
             })
             .move(0, 0)

diff --git a/cvat-canvas/src/typescript/shared.ts b/cvat-canvas/src/typescript/shared.ts
@@ -45,6 +45,7 @@ export interface DrawnState {
     shapeType: string;
     points?: number[];
     attributes: Record<number, string>;
+    descriptions: string[];
     zOrder?: number;
     pinned?: boolean;
     updated: number;

diff --git a/cvat-core/src/annotations-collection.js b/cvat-core/src/annotations-collection.js
@@ -722,6 +722,8 @@
                     checkObjectType('state occluded', state.occluded, 'boolean', null);
                     checkObjectType('state points', state.points, null, Array);
                     checkObjectType('state zOrder', state.zOrder, 'integer', null);
+                    checkObjectType('state descriptions', state.descriptions, null, Array);
+                    state.descriptions.forEach((desc) => checkObjectType('state description', desc, 'string'));
 
                     for (const coord of state.points) {
                         checkObjectType('point coordinate', coord, 'number', null);
@@ -736,6 +738,7 @@
                     if (state.objectType === 'shape') {
                         constructed.shapes.push({
                             attributes,
+                            descriptions: state.descriptions,
                             frame: state.frame,
                             group: 0,
                             label_id: state.label.id,
@@ -748,6 +751,7 @@
                     } else if (state.objectType === 'track') {
                         constructed.tracks.push({
                             attributes: attributes.filter((attr) => !labelAttributes[attr.spec_id].mutable),
+                            descriptions: state.descriptions,
                             frame: state.frame,
                             group: 0,
                             source: state.source,

diff --git a/cvat-core/src/annotations-objects.js b/cvat-core/src/annotations-objects.js
@@ -332,6 +332,14 @@
                 }
             }
 
+            if (updated.descriptions) {
+                if (!Array.isArray(data.descriptions) || data.descriptions.some((desc) => typeof desc !== 'string')) {
+                    throw new ArgumentError(
+                        `Descriptions are expected to be an array of strings but got ${data.descriptions}`,
+                    );
+                }
+            }
+
             if (updated.points) {
                 checkObjectType('points', data.points, null, Array);
                 checkNumberOfPoints(this.shapeType, data.points);
@@ -402,17 +410,7 @@
         }
 
         updateTimestamp(updated) {
-            const anyChanges = updated.label
-                || updated.attributes
-                || updated.points
-                || updated.outside
-                || updated.occluded
-                || updated.keyframe
-                || updated.zOrder
-                || updated.hidden
-                || updated.lock
-                || updated.pinned;
-
+            const anyChanges = Object.keys(updated).some((key) => !!updated[key]);
             if (anyChanges) {
                 this.updated = Date.now();
             }
@@ -446,11 +444,16 @@
         constructor(data, clientID, color, injection) {
             super(data, clientID, color, injection);
             this.frameMeta = injection.frameMeta;
+            this.descriptions = data.descriptions || [];
             this.hidden = false;
             this.pinned = true;
             this.shapeType = null;
         }
 
+        _saveDescriptions(descriptions) {
+            this.descriptions = [...descriptions];
+        }
+
         _savePinned(pinned, frame) {
             const undoPinned = this.pinned;
             const redoPinned = pinned;
@@ -533,6 +536,7 @@
                 zOrder: this.zOrder,
                 points: [...this.points],
                 attributes: { ...this.attributes },
+                descriptions: [...this.descriptions],
                 label: this.label,
                 group: this.groupObject,
                 color: this.color,
@@ -643,6 +647,10 @@
                 this._saveAttributes(data.attributes, frame);
             }
 
+            if (updated.descriptions) {
+                this._saveDescriptions(data.descriptions);
+            }
+
             if (updated.points && fittedPoints.length) {
                 this._savePoints(fittedPoints, frame);
             }
@@ -760,6 +768,7 @@
             return {
                 ...this.getPosition(frame, prev, next),
                 attributes: this.getAttributes(frame),
+                descriptions: [...this.descriptions],
                 group: this.groupObject,
                 objectType: ObjectType.TRACK,
                 shapeType: this.shapeType,
@@ -1204,6 +1213,10 @@
                 this._saveAttributes(data.attributes, frame);
             }
 
+            if (updated.descriptions) {
+                this._saveDescriptions(data.descriptions);
+            }
+
             if (updated.keyframe) {
                 this._saveKeyframe(frame, data.keyframe);
             }

diff --git a/cvat-core/src/object-state.js b/cvat-core/src/object-state.js
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -25,6 +25,7 @@ const { Source } = require('./enums');
             const data = {
                 label: null,
                 attributes: {},
+                descriptions: [],
 
                 points: null,
                 outside: null,
@@ -55,6 +56,7 @@ const { Source } = require('./enums');
                 value: function reset() {
                     this.label = false;
                     this.attributes = false;
+                    this.descriptions = false;
 
                     this.points = false;
                     this.outside = false;
@@ -70,6 +72,7 @@ const { Source } = require('./enums');
                     return reset;
                 },
                 writable: false,
+                enumerable: false,
             });
 
             Object.defineProperties(
@@ -353,6 +356,30 @@ const { Source } = require('./enums');
                             }
                         },
                     },
+                    descriptions: {
+                        /**
+                         * Additional text information displayed on canvas
+                         * @name descripttions
+                         * @type {string[]}
+                         * @memberof module:API.cvat.classes.ObjectState
+                         * @throws {module:API.cvat.exceptions.ArgumentError}
+                         * @instance
+                         */
+                        get: () => [...data.descriptions],
+                        set: (descriptions) => {
+                            if (
+                                !Array.isArray(descriptions)
+                                || descriptions.some((description) => typeof description !== 'string')
+                            ) {
+                                throw new ArgumentError(
+                                    `Descriptions are expected to be an array of strings but got ${data.descriptions}`,
+                                );
+                            }
+
+                            data.updateFlags.descriptions = true;
+                            data.descriptions = [...descriptions];
+                        },
+                    },
                 }),
             );
 
@@ -386,6 +413,12 @@ const { Source } = require('./enums');
             if (Array.isArray(serialized.points)) {
                 this.points = serialized.points;
             }
+            if (
+                Array.isArray(serialized.descriptions)
+                && serialized.descriptions.every((desc) => typeof desc === 'string')
+            ) {
+                this.descriptions = serialized.descriptions;
+            }
             if (typeof serialized.attributes === 'object') {
                 this.attributes = serialized.attributes;
             }
@@ -429,7 +462,7 @@ const { Source } = require('./enums');
     }
 
     // Updates element in collection which contains it
-    ObjectState.prototype.save.implementation = async function () {
+    ObjectState.prototype.save.implementation = function () {
         if (this.__internal && this.__internal.save) {
             return this.__internal.save();
         }
@@ -438,7 +471,7 @@ const { Source } = require('./enums');
     };
 
     // Delete element from a collection which contains it
-    ObjectState.prototype.delete.implementation = async function (frame, force) {
+    ObjectState.prototype.delete.implementation = function (frame, force) {
         if (this.__internal && this.__internal.delete) {
             if (!Number.isInteger(+frame) || +frame < 0) {
                 throw new ArgumentError('Frame argument must be a non negative integer');

diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts
@@ -2,7 +2,6 @@
 //
 // SPDX-License-Identifier: MIT
 
-import { MutableRefObject } from 'react';
 import {
     ActionCreator, AnyAction, Dispatch, Store,
 } from 'redux';
@@ -183,7 +182,6 @@ export enum AnnotationActionTypes {
     SAVE_LOGS_SUCCESS = 'SAVE_LOGS_SUCCESS',
     SAVE_LOGS_FAILED = 'SAVE_LOGS_FAILED',
     INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS',
-    SET_AI_TOOLS_REF = 'SET_AI_TOOLS_REF',
     GET_DATA_FAILED = 'GET_DATA_FAILED',
     SWITCH_REQUEST_REVIEW_DIALOG = 'SWITCH_REQUEST_REVIEW_DIALOG',
     SWITCH_SUBMIT_REVIEW_DIALOG = 'SWITCH_SUBMIT_REVIEW_DIALOG',
@@ -196,6 +194,7 @@ export enum AnnotationActionTypes {
     GET_CONTEXT_IMAGE = 'GET_CONTEXT_IMAGE',
     GET_CONTEXT_IMAGE_SUCCESS = 'GET_CONTEXT_IMAGE_SUCCESS',
     GET_CONTEXT_IMAGE_FAILED = 'GET_CONTEXT_IMAGE_FAILED',
+    SWITCH_NAVIGATION_BLOCKED = 'SWITCH_NAVIGATION_BLOCKED',
 }
 
 export function saveLogsAsync(): ThunkAction {
@@ -258,12 +257,14 @@ export function fetchAnnotationsAsync(): ThunkAction {
                 filters, frame, showAllInterpolationTracks, jobInstance,
             } = receiveAnnotationsParameters();
             const states = await jobInstance.annotations.get(frame, showAllInterpolationTracks, filters);
+            const history = await jobInstance.actions.get();
             const [minZ, maxZ] = computeZRange(states);
 
             dispatch({
                 type: AnnotationActionTypes.FETCH_ANNOTATIONS_SUCCESS,
                 payload: {
                     states,
+                    history,
                     minZ,
                     maxZ,
                 },
@@ -1460,15 +1461,6 @@ export function interactWithCanvas(activeInteractor: Model | OpenCVTool, activeL
     };
 }
 
-export function setAIToolsRef(ref: MutableRefObject<any>): AnyAction {
-    return {
-        type: AnnotationActionTypes.SET_AI_TOOLS_REF,
-        payload: {
-            aiToolsRef: ref,
-        },
-    };
-}
-
 export function repeatDrawShapeAsync(): ThunkAction {
     return async (dispatch: ActionCreator<Dispatch>): Promise<void> => {
         const {
@@ -1660,3 +1652,12 @@ export function getContextImageAsync(): ThunkAction {
         }
     };
 }
+
+export function switchNavigationBlocked(navigationBlocked: boolean): AnyAction {
+    return {
+        type: AnnotationActionTypes.SWITCH_NAVIGATION_BLOCKED,
+        payload: {
+            navigationBlocked,
+        },
+    };
+}