diff --git a/.gitignore b/.gitignore index c9591d564b..21d1070d30 100644 --- a/.gitignore +++ b/.gitignore @@ -177,4 +177,7 @@ test_evadb/ # tutorials tutorials/*.py *.pth + +# benchmark +.benchmarks eva.txt diff --git a/Jenkinsfile_weekly b/Jenkinsfile_weekly new file mode 100644 index 0000000000..f21012be8f --- /dev/null +++ b/Jenkinsfile_weekly @@ -0,0 +1,68 @@ +pipeline { + agent { + dockerfile { + filename 'docker/jenkins.Dockerfile' + args '--gpus all' + } + } + + options{ + buildDiscarder(logRotator(numToKeepStr: '8', daysToKeepStr: '20')) + } + + triggers { + cron '@weekly' + } + + stages { + + stage('Setup and Install Packages') { + parallel { + stage('Setup Virtual Environment') { + + steps { + sh '''python3 -m venv env37 + . env37/bin/activate + pip install --upgrade pip + pip install scikit-build + pip install cython + pip install -e ."[dev]" + ''' + } + } + stage('Generate Parser Files') { + + steps { + sh 'sh script/antlr4/generate_parser.sh' + } + } + } + } + + stage('CUDA GPU Check') { + + steps { + sh '''. env37/bin/activate + python3 -c "import torch; torch.cuda.current_device()" + ''' + } + } + + stage('Run Tests') { + + steps { + sh '''. env37/bin/activate + sh script/test/test_benchmark.sh + ''' + } + } + + stage('Coverage Check') { + + steps { + sh '''. env37/bin/activate + coveralls''' + } + } + } +} \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index 33ea6f9e7f..bf6deabc6b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,3 +5,4 @@ filterwarnings = markers = torchtest: marks tests that rely on torch (deselect with '-m "not torchtest"') + benchmark: marks tests that need to be benchmarked (deselect with '-m "not benchmark"') diff --git a/script/test/test.sh b/script/test/test.sh index 63246765bd..8fe52f8efe 100644 --- a/script/test/test.sh +++ b/script/test/test.sh @@ -28,7 +28,7 @@ fi # Run unit tests -PYTHONPATH=./ pytest test/ --cov-report term --cov-config=.coveragerc --cov=eva/ -s -v --log-level=WARNING ${1:-} +PYTHONPATH=./ pytest test/ --cov-report term --cov-config=.coveragerc --cov=eva/ -s -v --log-level=WARNING ${1:-} -m "not benchmark" test_code=$? if [ $test_code -ne 0 ]; then diff --git a/script/test/test_benchmark.sh b/script/test/test_benchmark.sh new file mode 100644 index 0000000000..45fd4c61e8 --- /dev/null +++ b/script/test/test_benchmark.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# generates a report for tests marked as benchmark tests + + +# temporarily remove __init__.py from root if it exists +if [ -f ./__init__.py ]; then + mv ./__init__.py ./__init__.py.bak +fi + +# Run black, isort, linter +sh script/formatting/pre-push.sh +return_code=$? +if [ $return_code -ne 0 ]; +then + exit $return_code +fi + +# Run only benchmark tests +PYTHONPATH=./ pytest test/ --cov-report term --cov-config=.coveragerc --cov=eva/ -s -v --log-level=WARNING ${1:-} -m "benchmark" +test_code=$? +if [ $test_code -ne 0 ]; +then + exit $test_code +fi + +# restore __init__.py if it exists +if [ -f ./__init__.py.bak ]; then + mv ./__init__.py.bak ./__init__.py +fi diff --git a/setup.py b/setup.py index bc0ac224b5..70f4576d78 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ def read(path, encoding="utf-8"): ] benchmark_libs = [ + "pytest-benchmark", ] doc_libs = [ diff --git a/test/benchmark_tests/conftest.py b/test/benchmark_tests/conftest.py new file mode 100644 index 0000000000..109488d74c --- /dev/null +++ b/test/benchmark_tests/conftest.py @@ -0,0 +1,36 @@ +# coding=utf-8 +# Copyright 2018-2022 EVA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from test.util import copy_sample_videos_to_upload_dir, file_remove, load_inbuilt_udfs + +import pytest + +from eva.catalog.catalog_manager import CatalogManager +from eva.server.command_handler import execute_query_fetch_all + + +@pytest.fixture(autouse=False) +def setup_pytorch_tests(): + file_remove("ua_detrac.mp4") + CatalogManager().reset() + copy_sample_videos_to_upload_dir() + query = """LOAD FILE 'ua_detrac.mp4' + INTO MyVideo;""" + execute_query_fetch_all(query) + query = """LOAD FILE 'mnist.mp4' + INTO MNIST;""" + execute_query_fetch_all(query) + load_inbuilt_udfs() + yield None + file_remove("ua_detrac.mp4") diff --git a/test/benchmark_tests/test_benchmark_pytorch.py b/test/benchmark_tests/test_benchmark_pytorch.py new file mode 100644 index 0000000000..68b2c8c569 --- /dev/null +++ b/test/benchmark_tests/test_benchmark_pytorch.py @@ -0,0 +1,164 @@ +# coding=utf-8 +# Copyright 2018-2022 EVA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys + +import mock +import pytest + +from eva.server.command_handler import execute_query_fetch_all + + +@pytest.mark.torchtest +@pytest.mark.benchmark( + warmup=False, + warmup_iterations=1, + min_rounds=1, +) +def test_should_run_pytorch_and_fastrcnn(benchmark, setup_pytorch_tests): + select_query = """SELECT FastRCNNObjectDetector(data) FROM MyVideo + WHERE id < 5;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert len(actual_batch) == 5 + + +@pytest.mark.torchtest +@pytest.mark.benchmark( + warmup=False, + warmup_iterations=1, + min_rounds=1, +) +def test_should_run_pytorch_and_ssd(benchmark, setup_pytorch_tests): + create_udf_query = """CREATE UDF IF NOT EXISTS SSDObjectDetector + INPUT (Frame_Array NDARRAY UINT8(3, 256, 256)) + OUTPUT (label NDARRAY STR(10)) + TYPE Classification + IMPL 'eva/udfs/ssd_object_detector.py'; + """ + execute_query_fetch_all(create_udf_query) + + select_query = """SELECT SSDObjectDetector(data) FROM MyVideo + WHERE id < 5;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert len(actual_batch) == 5 + # non-trivial test case + res = actual_batch.frames + for idx in res.index: + assert "car" in res["ssdobjectdetector.label"][idx] + + +@pytest.mark.torchtest +@pytest.mark.benchmark( + warmup=False, + warmup_iterations=1, + min_rounds=1, +) +def test_should_run_pytorch_and_facenet(benchmark, setup_pytorch_tests): + create_udf_query = """CREATE UDF IF NOT EXISTS FaceDetector + INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) + OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4), + scores NDARRAY FLOAT32(ANYDIM)) + TYPE FaceDetection + IMPL 'eva/udfs/face_detector.py'; + """ + execute_query_fetch_all(create_udf_query) + + select_query = """SELECT FaceDetector(data) FROM MyVideo + WHERE id < 5;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert len(actual_batch) == 5 + + +@pytest.mark.torchtest +@pytest.mark.benchmark( + warmup=False, + warmup_iterations=1, + min_rounds=1, +) +def test_should_run_pytorch_and_ocr(benchmark, setup_pytorch_tests): + create_udf_query = """CREATE UDF IF NOT EXISTS OCRExtractor + INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) + OUTPUT (labels NDARRAY STR(10), + bboxes NDARRAY FLOAT32(ANYDIM, 4), + scores NDARRAY FLOAT32(ANYDIM)) + TYPE OCRExtraction + IMPL 'eva/udfs/ocr_extractor.py'; + """ + execute_query_fetch_all(create_udf_query) + + select_query = """SELECT OCRExtractor(data) FROM MNIST + WHERE id >= 150 AND id < 155;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert len(actual_batch) == 5 + + # non-trivial test case for MNIST + res = actual_batch.frames + assert res["ocrextractor.labels"][0][0] == "4" + assert res["ocrextractor.scores"][2][0] > 0.9 + + +@pytest.mark.torchtest +@pytest.mark.benchmark( + warmup=False, + warmup_iterations=1, + min_rounds=1, +) +def test_should_run_pytorch_and_resnet50(benchmark, setup_pytorch_tests): + create_udf_query = """CREATE UDF IF NOT EXISTS FeatureExtractor + INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM)) + OUTPUT (features NDARRAY FLOAT32(ANYDIM)) + TYPE Classification + IMPL 'eva/udfs/feature_extractor.py'; + """ + execute_query_fetch_all(create_udf_query) + + select_query = """SELECT FeatureExtractor(data) FROM MyVideo + WHERE id < 5;""" + actual_batch = benchmark(execute_query_fetch_all, select_query) + assert len(actual_batch) == 5 + + # non-trivial test case for Resnet50 + res = actual_batch.frames + assert res["featureextractor.features"][0].shape == (1, 2048) + assert res["featureextractor.features"][0][0][0] > 0.3 + + +@pytest.mark.torchtest +@pytest.mark.benchmark( + warmup=False, + warmup_iterations=1, + min_rounds=1, +) +def test_should_raise_import_error_with_missing_torch(benchmark, setup_pytorch_tests): + with pytest.raises(ImportError): + with mock.patch.dict(sys.modules, {"torch": None}): + from eva.udfs.ssd_object_detector import SSDObjectDetector # noqa: F401 + + pass + + +@pytest.mark.torchtest +@pytest.mark.benchmark( + warmup=False, + warmup_iterations=1, + min_rounds=1, +) +def test_should_raise_import_error_with_missing_torchvision( + benchmark, setup_pytorch_tests +): + with pytest.raises(ImportError): + with mock.patch.dict(sys.modules, {"torchvision.transforms": None}): + from eva.udfs.ssd_object_detector import SSDObjectDetector # noqa: F401 + + pass diff --git a/test/util.py b/test/util.py index cd571aa9d8..5ea4d110dd 100644 --- a/test/util.py +++ b/test/util.py @@ -296,7 +296,10 @@ def copy_sample_videos_to_upload_dir(): def file_remove(path): - os.remove(os.path.join(upload_dir_from_config, path)) + try: + os.remove(os.path.join(upload_dir_from_config, path)) + except FileNotFoundError: + pass def create_dummy_batches(num_frames=NUM_FRAMES, filters=[], batch_size=10, start_id=0):