Analytics v2 (cvat-ai#127)

* Add revproxy dependency
* Fix formating of task.log, improve fields of server events.
* Mount kibana app into /analytics/*
* Add logs for revproxy and x-forwarded-user header.
* Added F3 shortcut for analytics/log-viewer.

analytics/docker-compose.yml

@@ -26,8 +26,6 @@ services:
       context: ./analytics/kibana
       args:
         ELK_VERSION: 6.4.0
-    ports:
-      - "5601:5601"
     depends_on: ['cvat_elasticsearch']
     restart: always
 
@@ -61,9 +59,11 @@ services:
 
   cvat:
     environment:
-      DJANGO_LOG_SERVER_HOST: "logstash"
+      DJANGO_LOG_SERVER_HOST: logstash
       DJANGO_LOG_SERVER_PORT: 5000
-      no_proxy: logstash,${no_proxy}
+      DJANGO_LOG_VIEWER_HOST: kibana
+      DJANGO_LOG_VIEWER_PORT: 5601
+      no_proxy: kibana,logstash,${no_proxy}
 
 volumes:
   cvat_events:

analytics/elasticsearch/Dockerfile

@@ -1,3 +1,4 @@
 ARG ELK_VERSION
 FROM docker.elastic.co/elasticsearch/elasticsearch-oss:${ELK_VERSION}
 COPY --chown=elasticsearch:elasticsearch elasticsearch.yml /usr/share/elasticsearch/config/
+

analytics/kibana/kibana.yml

@@ -1,4 +1,5 @@
 server.host: 0.0.0.0
 elasticsearch.url: http://elasticsearch:9200
-elasticsearch.requestHeadersWhitelist: [ cookie, authorization, x-forwarded-user ]
+elasticsearch.requestHeadersWhitelist: [ "cookie", "authorization", "x-forwarded-user" ]
 kibana.defaultAppId: "discover"
+server.basePath: /analytics

analytics/logstash/logstash.conf

@@ -54,10 +54,24 @@ filter {
       replace => { "type" => "client" }
     }
   } else if [logger_name] =~ /cvat.server/ {
-    # 1. Remove unnecessary field from it
-    # 2. Type it as server
+    # 1. Remove 'logger_name' field and create 'task' field
+    # 2. Remove unnecessary field from it
+    # 3. Type it as server
+    if [logger_name] =~ /cvat\.server\.task_[0-9]+/ {
+      mutate {
+        rename => { "logger_name" => "task" }
+        gsub => [ "task", "cvat.server.task_", "" ]
+      }
+
+      # Need to split the mutate because otherwise the conversion
+      # doesn't work.
+      mutate {
+        convert => { "task" => "integer" }
+      }
+    }
+
     prune {
-      blacklist_names => ["host", "port"]
+      blacklist_names => ["host", "port", "stack_info"]
     }
 
     mutate {

cvat/apps/documentation/static/documentation/js/shortcuts.js

@@ -5,7 +5,7 @@
  */
 
 Mousetrap.bind(window.cvat.config.shortkeys["open_help"].value, function() {
-    window.location.href = "/documentation/user_guide.html";
+    window.open("/documentation/user_guide.html");
 
     return false;
 });

cvat/apps/engine/annotation.py

@@ -22,7 +22,7 @@
 
 from . import models
 from .task import get_frame_path, get_image_meta_cache
-from .logging import task_logger, job_logger
+from .log import slogger
 
 ############################# Low Level server API
 
@@ -114,7 +114,7 @@ def save_task(tid, data):
 # pylint: disable=unused-argument
 def rq_handler(job, exc_type, exc_value, traceback):
     tid = job.id.split('/')[1]
-    task_logger[tid].error("dump annotation error was occured", exc_info=True)
+    slogger.task[tid].error("dump annotation error was occured", exc_info=True)
 
 ##################################################
 
@@ -407,7 +407,7 @@ def __init__(self, db_job):
 
         # pylint: disable=bad-continuation
         self.db_job = db_job
-        self.logger = job_logger[db_job.id]
+        self.logger = slogger.job[db_job.id]
         self.db_labels = {db_label.id:db_label
             for db_label in db_job.segment.task.label_set.all()}
         self.db_attributes = {db_attr.id:db_attr

cvat/apps/engine/logging.py → cvat/apps/engine/log.py

@@ -4,9 +4,8 @@
 
 import os
 import logging
-from . import models
 from cvat.settings.base import LOGGING
-from cvat.apps.engine.models import Job, Task
+from .models import Job, Task
 
 def _get_task(tid):
     try:
@@ -16,7 +15,7 @@ def _get_task(tid):
 
 def _get_job(jid):
     try:
-        return models.Job.objects.select_related("segment__task").get(id=jid)
+        return Job.objects.select_related("segment__task").get(id=jid)
     except Exception:
         raise Exception('{} key must be a job identifier'.format(jid))
 
@@ -34,6 +33,8 @@ def _create_task_logger(self, tid):
 
         logger = logging.getLogger('cvat.server.task_{}'.format(tid))
         server_file = logging.FileHandler(filename=task.get_log_path())
+        formatter = logging.Formatter(LOGGING['formatters']['standard']['format'])
+        server_file.setFormatter(formatter)
         logger.addHandler(server_file)
 
         return logger
@@ -49,7 +50,7 @@ def __getitem__(self, jid):
 
     def _get_task_logger(self, jid):
         job = _get_job(jid)
-        return task_logger[job.segment.task.id]
+        return slogger.task[job.segment.task.id]
 
 class TaskClientLoggerStorage:
     def __init__(self):
@@ -79,10 +80,21 @@ def __getitem__(self, jid):
 
     def _get_task_logger(self, jid):
         job = _get_job(jid)
-        return task_client_logger[job.segment.task.id]
-
-task_logger = TaskLoggerStorage()
-job_logger = JobLoggerStorage()
-global_logger = logging.getLogger('cvat.server')
-job_client_logger = JobClientLoggerStorage()
-task_client_logger = TaskClientLoggerStorage()
+        return clogger.task[job.segment.task.id]
+
+class dotdict(dict):
+    """dot.notation access to dictionary attributes"""
+    __getattr__ = dict.get
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+clogger = dotdict({
+    'task': TaskClientLoggerStorage(),
+    'job': JobClientLoggerStorage()
+})
+
+slogger = dotdict({
+    'task': TaskLoggerStorage(),
+    'job': JobLoggerStorage(),
+    'glob': logging.getLogger('cvat.server'),
+})

cvat/apps/engine/static/engine/js/userConfig.js

@@ -192,6 +192,12 @@ class Config {
                 description: "open settings window "
             },
 
+            open_analytics: {
+                value: "f3",
+                view_value: "F3",
+                description: "open analytics window"
+            },
+
             save_work: {
                 value: "ctrl+s",
                 view_value: "Ctrl + S",

cvat/apps/engine/task.py

@@ -3,16 +3,12 @@
 #
 # SPDX-License-Identifier: MIT
 
-import csv
 import os
-import re
-import rq
 import sys
+import rq
 import shlex
-import logging
 import shutil
 import tempfile
-from io import StringIO
 from PIL import Image
 from traceback import print_exception
 from ast import literal_eval
@@ -30,7 +26,7 @@
 from distutils.dir_util import copy_tree
 
 from . import models
-from .logging import task_logger, job_logger, global_logger
+from .log import slogger
 
 ############################# Low Level server API
 
@@ -579,7 +575,8 @@ def _save_task_to_db(db_task, task_params):
     for x in range(0, db_task.size, segment_step):
         start_frame = x
         stop_frame = min(x + task_params['segment'] - 1, db_task.size - 1)
-        global_logger.info("New segment for task #{}: start_frame = {}, stop_frame = {}".format(db_task.id, start_frame, stop_frame))
+        slogger.glob.info("New segment for task #{}: start_frame = {}, \
+            stop_frame = {}".format(db_task.id, start_frame, stop_frame))
 
         db_segment = models.Segment()
         db_segment.task = db_task
@@ -613,7 +610,7 @@ def raise_exception(images, dirs, videos, archives):
         raise Exception('Only one archive, one video or many images can be dowloaded simultaneously. \
             {} image(s), {} dir(s), {} video(s), {} archive(s) found'.format(images, dirs, videos, archives))
 
-    global_logger.info("create task #{}".format(tid))
+    slogger.glob.info("create task #{}".format(tid))
     job = rq.get_current_job()
 
     db_task = models.Task.objects.select_for_update().get(pk=tid)
@@ -657,13 +654,13 @@ def raise_exception(images, dirs, videos, archives):
     }
     task_params['overlap'] = int(params.get('overlap_size', 5 if task_params['mode'] == 'interpolation' else 0))
     task_params['overlap'] = min(task_params['overlap'], task_params['segment'] - 1)
-    global_logger.info("Task #{} parameters: {}".format(tid, task_params))
+    slogger.glob.info("Task #{} parameters: {}".format(tid, task_params))
 
     if task_params['mode'] == 'interpolation':
         _find_and_extract_video(upload_dir, output_dir, db_task, task_params['compress'], task_params['flip'], job)
     else:
         _find_and_compress_images(upload_dir, output_dir, db_task, task_params['compress'], task_params['flip'], job)
-    global_logger.info("Founded frames {} for task #{}".format(db_task.size, tid))
+    slogger.glob.info("Founded frames {} for task #{}".format(db_task.size, tid))
 
     job.meta['status'] = 'Task is being saved in database'
     job.save_meta()

cvat/apps/engine/urls.py

@@ -9,7 +9,7 @@
 urlpatterns = [
     path('', views.dispatch_request),
     path('create/task', views.create_task),
-    path('get/task/<str:tid>/frame/<int:frame>', views.get_frame),
+    path('get/task/<int:tid>/frame/<int:frame>', views.get_frame),
     path('check/task/<int:tid>', views.check_task),
     path('delete/task/<int:tid>', views.delete_task),
     path('update/task/<int:tid>', views.update_task),