Merge pull request cvat-ai#5980 from opencv/hotfix-2.4.1

nmanovic · web-flow · commit a7f7ff1254f4 · 2023-04-05T21:36:22.000+03:00
Hotfix for v2.4.0 -&gt; Release v2.4.1
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## \[2.4.1] - 2023-04-05
+### Fixed
+- Optimized annotation fetching up to 10 times (<https://github.com/opencv/cvat/pull/5974>)
+- Incorrect calculation of working time in analytics (<https://github.com/opencv/cvat/pull/5973>)
+
 ## \[2.4.0] - 2023-03-16
 ### Added
 - \[SDK\] An arg to wait for data processing in the task data uploading function
diff --git a/cvat/__init__.py b/cvat/__init__.py
@@ -4,6 +4,6 @@
 
 from cvat.utils.version import get_version
 
-VERSION = (2, 4, 0, 'final', 0)
+VERSION = (2, 4, 1, 'final', 0)
 
 __version__ = get_version(VERSION)
diff --git a/cvat/apps/dataset_manager/task.py b/cvat/apps/dataset_manager/task.py
@@ -399,7 +399,7 @@ def _init_tags_from_db(self):
             self._extend_attributes(db_tag.labeledimageattributeval_set,
                 self.db_attributes[db_tag.label_id]["all"].values())
 
-        serializer = serializers.LabeledImageSerializer(db_tags, many=True)
+        serializer = serializers.LabeledImageSerializerFromDB(db_tags, many=True)
         self.ir_data.tags = serializer.data
 
     def _init_shapes_from_db(self):
@@ -453,7 +453,7 @@ def _init_shapes_from_db(self):
         for shape_id, shape_elements in elements.items():
             shapes[shape_id].elements = shape_elements
 
-        serializer = serializers.LabeledShapeSerializer(list(shapes.values()), many=True)
+        serializer = serializers.LabeledShapeSerializerFromDB(list(shapes.values()), many=True)
         self.ir_data.shapes = serializer.data
 
     def _init_tracks_from_db(self):
@@ -546,7 +546,7 @@ def _init_tracks_from_db(self):
         for track_id, track_elements in elements.items():
             tracks[track_id].elements = track_elements
 
-        serializer = serializers.LabeledTrackSerializer(list(tracks.values()), many=True)
+        serializer = serializers.LabeledTrackSerializerFromDB(list(tracks.values()), many=True)
         self.ir_data.tracks = serializer.data
 
     def _init_version_from_db(self):
diff --git a/cvat/apps/engine/mixins.py b/cvat/apps/engine/mixins.py
@@ -16,7 +16,7 @@
 
 from cvat.apps.engine.models import Location
 from cvat.apps.engine.location import StorageType, get_location_configuration
-from cvat.apps.engine.serializers import DataSerializer, LabeledDataSerializer
+from cvat.apps.engine.serializers import DataSerializer
 from cvat.apps.webhooks.signals import signal_update, signal_create, signal_delete
 
 class TusFile:
@@ -278,9 +278,7 @@ def export_annotations(self, request, db_obj, export_func, callback, get_data=No
             return Response("Format is not specified",status=status.HTTP_400_BAD_REQUEST)
 
         data = get_data(self._object.pk)
-        serializer = LabeledDataSerializer(data=data)
-        if serializer.is_valid(raise_exception=True):
-            return Response(serializer.data)
+        return Response(data)
 
     def import_annotations(self, request, db_obj, import_func, rq_func, rq_id):
         is_tus_request = request.headers.get('Upload-Length', None) is not None or \
diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py
@@ -1189,7 +1189,6 @@ def run_child_validation(self, data):
 
         raise exceptions.ValidationError(errors)
 
-
 class ShapeSerializer(serializers.Serializer):
     type = serializers.ChoiceField(choices=models.ShapeType.choices())
     occluded = serializers.BooleanField(default=False)
@@ -1207,6 +1206,63 @@ class SubLabeledShapeSerializer(ShapeSerializer, AnnotationSerializer):
 class LabeledShapeSerializer(SubLabeledShapeSerializer):
     elements = SubLabeledShapeSerializer(many=True, required=False)
 
+def _convert_annotation(obj, keys):
+    return OrderedDict([(key, obj[key]) for key in keys])
+
+def _convert_attributes(attr_set):
+    attr_keys = ['spec_id', 'value']
+    return [
+        OrderedDict([(key, attr[key]) for key in attr_keys]) for attr in attr_set
+    ]
+
+class LabeledImageSerializerFromDB(serializers.BaseSerializer):
+    # Use this serializer to export data from the database
+    # Because default DRF serializer is too slow on huge collections
+    def to_representation(self, instance):
+        def convert_tag(tag):
+            result = _convert_annotation(tag, ['id', 'label_id', 'frame', 'group', 'source'])
+            result['attributes'] = _convert_attributes(tag['labeledimageattributeval_set'])
+            return result
+
+        return convert_tag(instance)
+
+class LabeledShapeSerializerFromDB(serializers.BaseSerializer):
+    # Use this serializer to export data from the database
+    # Because default DRF serializer is too slow on huge collections
+    def to_representation(self, instance):
+        def convert_shape(shape):
+            result = _convert_annotation(shape, [
+                'id', 'label_id', 'type', 'frame', 'group', 'source',
+                'occluded', 'outside', 'z_order', 'rotation', 'points',
+            ])
+            result['attributes'] = _convert_attributes(shape['labeledshapeattributeval_set'])
+            if shape.get('elements', None) is not None and shape['parent'] is None:
+                result['elements'] = [convert_shape(element) for element in shape['elements']]
+            return result
+
+        return convert_shape(instance)
+
+class LabeledTrackSerializerFromDB(serializers.BaseSerializer):
+    # Use this serializer to export data from the database
+    # Because default DRF serializer is too slow on huge collections
+    def to_representation(self, instance):
+        def convert_track(track):
+            shape_keys = [
+                'id', 'type', 'frame', 'occluded', 'outside', 'z_order',
+                'rotation', 'points', 'trackedshapeattributeval_set',
+            ]
+            result = _convert_annotation(track, ['id', 'label_id', 'frame', 'group', 'source'])
+            result['shapes'] = [_convert_annotation(shape, shape_keys) for shape in track['trackedshape_set']]
+            result['attributes'] = _convert_attributes(track['labeledtrackattributeval_set'])
+            for shape in result['shapes']:
+                shape['attributes'] = _convert_attributes(shape['trackedshapeattributeval_set'])
+                shape.pop('trackedshapeattributeval_set', None)
+            if track.get('elements', None) is not None and track['parent'] is None:
+                result['elements'] = [convert_track(element) for element in track['elements']]
+            return result
+
+        return convert_track(instance)
+
 class TrackedShapeSerializer(ShapeSerializer):
     id = serializers.IntegerField(default=None, allow_null=True)
     frame = serializers.IntegerField(min_value=0)
diff --git a/cvat/apps/events/serializers.py b/cvat/apps/events/serializers.py
@@ -30,7 +30,8 @@ class EventSerializer(serializers.Serializer):
 class ClientEventsSerializer(serializers.Serializer):
     events = EventSerializer(many=True, default=[])
     timestamp = serializers.DateTimeField()
-    _TIME_THRESHOLD = 100 # seconds
+    _TIME_THRESHOLD = datetime.timedelta(seconds=100)
+    _WORKING_TIME_RESOLUTION = datetime.timedelta(milliseconds=1)
 
     def to_internal_value(self, data):
         request = self.context.get("request")
@@ -47,12 +48,12 @@ def to_internal_value(self, data):
             timestamp = datetime_parser.isoparse(event['timestamp'])
             if last_timestamp:
                 t_diff = timestamp - last_timestamp
-                if t_diff.seconds < self._TIME_THRESHOLD:
+                if t_diff < self._TIME_THRESHOLD:
                     payload = event.get('payload', {})
                     if payload:
                         payload = json.loads(payload)
 
-                    payload['working_time'] = t_diff.microseconds // 1000
+                    payload['working_time'] = t_diff // self._WORKING_TIME_RESOLUTION
                     payload['username'] = request.user.username
                     event['payload'] = json.dumps(payload)
 
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -25,7 +25,7 @@ services:
 
   cvat_server:
     container_name: cvat_server
-    image: cvat/server:${CVAT_VERSION:-v2.4.0}
+    image: cvat/server:${CVAT_VERSION:-v2.4.1}
     restart: always
     depends_on:
       - cvat_redis
@@ -63,7 +63,7 @@ services:
 
   cvat_utils:
     container_name: cvat_utils
-    image: cvat/server:${CVAT_VERSION:-v2.4.0}
+    image: cvat/server:${CVAT_VERSION:-v2.4.1}
     restart: always
     depends_on:
       - cvat_redis
@@ -84,7 +84,7 @@ services:
 
   cvat_worker_import:
     container_name: cvat_worker_import
-    image: cvat/server:${CVAT_VERSION:-v2.4.0}
+    image: cvat/server:${CVAT_VERSION:-v2.4.1}
     restart: always
     depends_on:
       - cvat_redis
@@ -106,7 +106,7 @@ services:
 
   cvat_worker_export:
     container_name: cvat_worker_export
-    image: cvat/server:${CVAT_VERSION:-v2.4.0}
+    image: cvat/server:${CVAT_VERSION:-v2.4.1}
     restart: always
     depends_on:
       - cvat_redis
@@ -127,7 +127,7 @@ services:
 
   cvat_worker_annotation:
     container_name: cvat_worker_annotation
-    image: cvat/server:${CVAT_VERSION:-v2.4.0}
+    image: cvat/server:${CVAT_VERSION:-v2.4.1}
     restart: always
     depends_on:
       - cvat_redis
@@ -148,7 +148,7 @@ services:
 
   cvat_worker_webhooks:
     container_name: cvat_worker_webhooks
-    image: cvat/server:${CVAT_VERSION:-v2.4.0}
+    image: cvat/server:${CVAT_VERSION:-v2.4.1}
     restart: always
     depends_on:
       - cvat_redis
@@ -169,7 +169,7 @@ services:
 
   cvat_ui:
     container_name: cvat_ui
-    image: cvat/ui:${CVAT_VERSION:-v2.4.0}
+    image: cvat/ui:${CVAT_VERSION:-v2.4.1}
     restart: always
     depends_on:
       - cvat_server
diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml
@@ -75,7 +75,7 @@ cvat:
       additionalVolumeMounts: []
     replicas: 1
     image: cvat/server
-    tag: v2.4.0
+    tag: v2.4.1
     imagePullPolicy: Always
     permissionFix:
       enabled: true
@@ -95,7 +95,7 @@ cvat:
   frontend:
     replicas: 1
     image: cvat/ui
-    tag: v2.4.0
+    tag: v2.4.1
     imagePullPolicy: Always
     labels: {}
     #  test: test
