Benchmarking config (georgia-tech-db#482)

* Bechmarking tests added in a separate file

* Formatting added

* Disable benchmarking in the current job

* New perf job config

* re arranged the folder structure

* Fixed some comments

Co-authored-by: Rajveer Bachkaniwala <[email protected]>

.gitignore

@@ -177,4 +177,7 @@ test_evadb/
 # tutorials
 tutorials/*.py
 *.pth
+
+# benchmark
+.benchmarks
 eva.txt

Jenkinsfile_weekly

@@ -0,0 +1,68 @@
+pipeline {
+  agent {
+      dockerfile {
+        filename 'docker/jenkins.Dockerfile'
+        args '--gpus all'
+      }
+    }
+
+  options{
+     buildDiscarder(logRotator(numToKeepStr: '8', daysToKeepStr: '20'))
+  }
+
+  triggers {
+    cron '@weekly'
+  }
+
+  stages {
+
+    stage('Setup and Install Packages') {
+      parallel {
+        stage('Setup Virtual Environment') {
+
+          steps {
+            sh '''python3 -m venv env37
+                  . env37/bin/activate
+                  pip install --upgrade pip
+                  pip install scikit-build
+                  pip install cython
+                  pip install -e ."[dev]"
+              '''
+          }
+        }
+        stage('Generate Parser Files') {
+
+          steps {
+            sh 'sh script/antlr4/generate_parser.sh'
+          }
+        }
+      }
+    }
+
+    stage('CUDA GPU Check') {
+
+      steps {
+          sh '''. env37/bin/activate
+                python3 -c "import torch; torch.cuda.current_device()"
+             '''
+      }
+    }
+
+    stage('Run Tests') {
+
+      steps {
+        sh '''. env37/bin/activate
+              sh script/test/test_benchmark.sh
+           '''
+       }
+     }
+
+    stage('Coverage Check') {
+
+      steps {
+        sh '''. env37/bin/activate
+          coveralls'''
+      }
+    }
+  }
+}

pytest.ini

@@ -5,3 +5,4 @@ filterwarnings =
 
 markers =
     torchtest: marks tests that rely on torch (deselect with '-m "not torchtest"')
+    benchmark: marks tests that need to be benchmarked (deselect with '-m "not benchmark"')

script/test/test.sh

@@ -28,7 +28,7 @@ fi
 
 
 # Run unit tests
-PYTHONPATH=./ pytest test/ --cov-report term --cov-config=.coveragerc --cov=eva/ -s -v --log-level=WARNING ${1:-}
+PYTHONPATH=./ pytest test/ --cov-report term --cov-config=.coveragerc --cov=eva/ -s -v --log-level=WARNING ${1:-} -m "not benchmark"
 test_code=$?
 if [ $test_code -ne 0 ];
 then

script/test/test_benchmark.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+# generates a report for tests marked as benchmark tests
+
+
+# temporarily remove __init__.py from root if it exists
+if [ -f ./__init__.py ]; then
+   mv ./__init__.py ./__init__.py.bak
+fi
+
+# Run black, isort, linter 
+sh script/formatting/pre-push.sh
+return_code=$?
+if [ $return_code -ne 0 ];
+then
+    exit $return_code
+fi
+
+# Run only benchmark tests
+PYTHONPATH=./ pytest test/ --cov-report term --cov-config=.coveragerc --cov=eva/ -s -v --log-level=WARNING ${1:-} -m "benchmark"
+test_code=$?
+if [ $test_code -ne 0 ];
+then
+    exit $test_code
+fi
+
+# restore __init__.py if it exists
+if [ -f ./__init__.py.bak ]; then
+    mv ./__init__.py.bak ./__init__.py
+fi

setup.py

@@ -79,6 +79,7 @@ def read(path, encoding="utf-8"):
 ]
 
 benchmark_libs = [
+    "pytest-benchmark",
 ]
 
 doc_libs = [

test/benchmark_tests/conftest.py

@@ -0,0 +1,36 @@
+# coding=utf-8
+# Copyright 2018-2022 EVA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from test.util import copy_sample_videos_to_upload_dir, file_remove, load_inbuilt_udfs
+
+import pytest
+
+from eva.catalog.catalog_manager import CatalogManager
+from eva.server.command_handler import execute_query_fetch_all
+
+
+@pytest.fixture(autouse=False)
+def setup_pytorch_tests():
+    file_remove("ua_detrac.mp4")
+    CatalogManager().reset()
+    copy_sample_videos_to_upload_dir()
+    query = """LOAD FILE 'ua_detrac.mp4'
+                INTO MyVideo;"""
+    execute_query_fetch_all(query)
+    query = """LOAD FILE 'mnist.mp4'
+                INTO MNIST;"""
+    execute_query_fetch_all(query)
+    load_inbuilt_udfs()
+    yield None
+    file_remove("ua_detrac.mp4")

test/benchmark_tests/test_benchmark_pytorch.py

@@ -0,0 +1,164 @@
+# coding=utf-8
+# Copyright 2018-2022 EVA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+import mock
+import pytest
+
+from eva.server.command_handler import execute_query_fetch_all
+
+
+@pytest.mark.torchtest
+@pytest.mark.benchmark(
+    warmup=False,
+    warmup_iterations=1,
+    min_rounds=1,
+)
+def test_should_run_pytorch_and_fastrcnn(benchmark, setup_pytorch_tests):
+    select_query = """SELECT FastRCNNObjectDetector(data) FROM MyVideo
+                    WHERE id < 5;"""
+    actual_batch = benchmark(execute_query_fetch_all, select_query)
+    assert len(actual_batch) == 5
+
+
+@pytest.mark.torchtest
+@pytest.mark.benchmark(
+    warmup=False,
+    warmup_iterations=1,
+    min_rounds=1,
+)
+def test_should_run_pytorch_and_ssd(benchmark, setup_pytorch_tests):
+    create_udf_query = """CREATE UDF IF NOT EXISTS SSDObjectDetector
+                INPUT  (Frame_Array NDARRAY UINT8(3, 256, 256))
+                OUTPUT (label NDARRAY STR(10))
+                TYPE  Classification
+                IMPL  'eva/udfs/ssd_object_detector.py';
+    """
+    execute_query_fetch_all(create_udf_query)
+
+    select_query = """SELECT SSDObjectDetector(data) FROM MyVideo
+                    WHERE id < 5;"""
+    actual_batch = benchmark(execute_query_fetch_all, select_query)
+    assert len(actual_batch) == 5
+    # non-trivial test case
+    res = actual_batch.frames
+    for idx in res.index:
+        assert "car" in res["ssdobjectdetector.label"][idx]
+
+
+@pytest.mark.torchtest
+@pytest.mark.benchmark(
+    warmup=False,
+    warmup_iterations=1,
+    min_rounds=1,
+)
+def test_should_run_pytorch_and_facenet(benchmark, setup_pytorch_tests):
+    create_udf_query = """CREATE UDF IF NOT EXISTS FaceDetector
+                INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
+                OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4),
+                        scores NDARRAY FLOAT32(ANYDIM))
+                TYPE  FaceDetection
+                IMPL  'eva/udfs/face_detector.py';
+    """
+    execute_query_fetch_all(create_udf_query)
+
+    select_query = """SELECT FaceDetector(data) FROM MyVideo
+                    WHERE id < 5;"""
+    actual_batch = benchmark(execute_query_fetch_all, select_query)
+    assert len(actual_batch) == 5
+
+
+@pytest.mark.torchtest
+@pytest.mark.benchmark(
+    warmup=False,
+    warmup_iterations=1,
+    min_rounds=1,
+)
+def test_should_run_pytorch_and_ocr(benchmark, setup_pytorch_tests):
+    create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor
+                INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
+                OUTPUT (labels NDARRAY STR(10),
+                        bboxes NDARRAY FLOAT32(ANYDIM, 4),
+                        scores NDARRAY FLOAT32(ANYDIM))
+                TYPE  OCRExtraction
+                IMPL  'eva/udfs/ocr_extractor.py';
+    """
+    execute_query_fetch_all(create_udf_query)
+
+    select_query = """SELECT OCRExtractor(data) FROM MNIST
+                    WHERE id >= 150 AND id < 155;"""
+    actual_batch = benchmark(execute_query_fetch_all, select_query)
+    assert len(actual_batch) == 5
+
+    # non-trivial test case for MNIST
+    res = actual_batch.frames
+    assert res["ocrextractor.labels"][0][0] == "4"
+    assert res["ocrextractor.scores"][2][0] > 0.9
+
+
+@pytest.mark.torchtest
+@pytest.mark.benchmark(
+    warmup=False,
+    warmup_iterations=1,
+    min_rounds=1,
+)
+def test_should_run_pytorch_and_resnet50(benchmark, setup_pytorch_tests):
+    create_udf_query = """CREATE UDF IF NOT EXISTS FeatureExtractor
+                INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
+                OUTPUT (features NDARRAY FLOAT32(ANYDIM))
+                TYPE  Classification
+                IMPL  'eva/udfs/feature_extractor.py';
+    """
+    execute_query_fetch_all(create_udf_query)
+
+    select_query = """SELECT FeatureExtractor(data) FROM MyVideo
+                    WHERE id < 5;"""
+    actual_batch = benchmark(execute_query_fetch_all, select_query)
+    assert len(actual_batch) == 5
+
+    # non-trivial test case for Resnet50
+    res = actual_batch.frames
+    assert res["featureextractor.features"][0].shape == (1, 2048)
+    assert res["featureextractor.features"][0][0][0] > 0.3
+
+
+@pytest.mark.torchtest
+@pytest.mark.benchmark(
+    warmup=False,
+    warmup_iterations=1,
+    min_rounds=1,
+)
+def test_should_raise_import_error_with_missing_torch(benchmark, setup_pytorch_tests):
+    with pytest.raises(ImportError):
+        with mock.patch.dict(sys.modules, {"torch": None}):
+            from eva.udfs.ssd_object_detector import SSDObjectDetector  # noqa: F401
+
+            pass
+
+
+@pytest.mark.torchtest
+@pytest.mark.benchmark(
+    warmup=False,
+    warmup_iterations=1,
+    min_rounds=1,
+)
+def test_should_raise_import_error_with_missing_torchvision(
+    benchmark, setup_pytorch_tests
+):
+    with pytest.raises(ImportError):
+        with mock.patch.dict(sys.modules, {"torchvision.transforms": None}):
+            from eva.udfs.ssd_object_detector import SSDObjectDetector  # noqa: F401
+
+            pass

test/util.py

@@ -296,7 +296,10 @@ def copy_sample_videos_to_upload_dir():
 
 
 def file_remove(path):
-    os.remove(os.path.join(upload_dir_from_config, path))
+    try:
+        os.remove(os.path.join(upload_dir_from_config, path))
+    except FileNotFoundError:
+        pass
 
 
 def create_dummy_batches(num_frames=NUM_FRAMES, filters=[], batch_size=10, start_id=0):