feat: server tests (georgia-tech-db#562)

This PR:

- [x] Adds support for stopping and restarting the server to check the catalog in: 

https://github.com/georgia-tech-db/eva/blob/0dc98763779c13c86bd5a3edafa8006220b9f36e/test/integration_tests/test_show_info_executor.py#L88-L90

This will prevent issues like georgia-tech-db#531

- [x] Takes care of the `hasattr` refactoring for singleton classes in georgia-tech-db#416

- [x] Patching `evaluate` in the test to reduce the time taken to run the testcase and using a side-effect to count the number of `evaluate` invocations:

https://github.com/georgia-tech-db/eva/blob/0dc98763779c13c86bd5a3edafa8006220b9f36e/test/integration_tests/test_optimizer_rules.py#L45

- [x] Updates to benchmarking script. Disabled benchmarking on CircleCI as it depends on server workload.

https://github.com/georgia-tech-db/eva/blob/0dc98763779c13c86bd5a3edafa8006220b9f36e/script/test/test_benchmark.sh#L19-L35

.circleci/config.yml

@@ -67,14 +67,13 @@ jobs:
           paths:
             - /home/circleci/.cache/torch/hub/
             - /home/circleci/.EasyOCR/
-
 
   Windows:
     executor: win/default
     parameters:
       v:
         type: string
-        default: "3.7" 
+        default: "3.10" 
     steps:
       - checkout
 

.gitignore

@@ -177,7 +177,9 @@ test_evadb/
 # tutorials
 tutorials/*.py
 *.pth
+*.pt
 
 # benchmark
 .benchmarks
 eva.txt
+prof/

eva/catalog/catalog_manager.py

@@ -42,10 +42,8 @@
 
 
 class CatalogManager(object):
-    _instance = None
-
     def __new__(cls):
-        if cls._instance is None:
+        if not hasattr(cls, "_instance"):
             cls._instance = super(CatalogManager, cls).__new__(cls)
 
             cls._instance._bootstrap_catalog()

eva/catalog/models/base_model.py

@@ -107,8 +107,8 @@ def init_db():
     if not database_exists(engine.url):
         logger.info("Database does not exist, creating database.")
         create_database(engine.url)
-    logger.info("Creating tables")
-    BaseModel.metadata.create_all()
+        logger.info("Creating tables")
+        BaseModel.metadata.create_all()
 
 
 def drop_db():

eva/catalog/sql_config.py

@@ -27,8 +27,6 @@ class SQLConfig:
         _instance: stores the singleton instance of the class.
     """
 
-    _instance = None
-
     def __new__(cls):
         """Overrides the default __new__ method.
 
@@ -38,7 +36,7 @@ def __new__(cls):
         Returns:
             An instance of the class.
         """
-        if cls._instance is None:
+        if not hasattr(cls, "_instance"):
             cls._instance = super(SQLConfig, cls).__new__(cls)
         return cls._instance
 

eva/configuration/configuration_manager.py

@@ -25,11 +25,10 @@
 
 
 class ConfigurationManager(object):
-    _instance = None
     _yml_path = EVA_DEFAULT_DIR / EVA_CONFIG_FILE
 
     def __new__(cls):
-        if cls._instance is None:
+        if not hasattr(cls, "_instance"):
             cls._instance = super(ConfigurationManager, cls).__new__(cls)
             cls._create_if_not_exists()
 

eva/eva_server.py

@@ -12,9 +12,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import argparse
 import asyncio
 import sys
 from os.path import abspath, dirname, join
+from signal import SIGTERM
+
+from psutil import process_iter
 
 from eva.utils.logging_manager import logger
 
@@ -58,10 +62,46 @@ def eva():
         logger.critical(e)
 
 
+def stop_server():
+    """
+    Stop the eva server
+    """
+    for proc in process_iter():
+        if proc.name() == "eva_server":
+            proc.send_signal(SIGTERM)
+
+    return 0
+
+
 def main():
-    mode = ConfigurationManager().get_value("core", "mode")
-    init_builtin_udfs(mode=mode)
-    eva()
+    parser = argparse.ArgumentParser(description="EVA Server")
+
+    parser.add_argument(
+        "--start",
+        help="start server",
+        action="store_true",
+        default=True,
+    )
+
+    parser.add_argument(
+        "--stop",
+        help="stop server",
+        action="store_true",
+        default=False,
+    )
+
+    ## PARSE ARGS
+    args, unknown = parser.parse_known_args()
+
+    # Stop server
+    if args.stop:
+        return stop_server()
+
+    # Start server
+    if args.start:
+        mode = ConfigurationManager().get_value("core", "mode")
+        init_builtin_udfs(mode=mode)
+        eva()
 
 
 if __name__ == "__main__":

eva/executor/execution_context.py

@@ -29,10 +29,8 @@ class Context:
     if using horovod: current rank etc.
     """
 
-    _instance = None
-
     def __new__(cls):
-        if cls._instance is None:
+        if not hasattr(cls, "_instance"):
             cls._instance = super(Context, cls).__new__(cls)
         return cls._instance
 

eva/parser/lark_parser.py

@@ -24,11 +24,10 @@ class LarkParser(object):
     Parser for EVA QL based on Lark
     """
 
-    _instance = None
     _parser = None
 
     def __new__(cls):
-        if cls._instance is None:
+        if not hasattr(cls, "_instance"):
             cls._instance = super(LarkParser, cls).__new__(cls)
         return cls._instance
 

eva/parser/parser.py

@@ -20,11 +20,10 @@ class Parser(object):
     Parser based on EVAQL grammar: eva.lark
     """
 
-    _instance = None
     _lark_parser = None
 
     def __new__(cls):
-        if cls._instance is None:
+        if not hasattr(cls, "_instance"):
             cls._instance = super(Parser, cls).__new__(cls)
             cls._instance._initialized = False
         return cls._instance

script/test/test_benchmark.sh

@@ -2,7 +2,6 @@
 
 # generates a report for tests marked as benchmark tests
 
-
 # temporarily remove __init__.py from root if it exists
 if [ -f ./__init__.py ]; then
    mv ./__init__.py ./__init__.py.bak
@@ -16,12 +15,27 @@ then
     exit $return_code
 fi
 
+
 # Run only benchmark tests
-PYTHONPATH=./ pytest test/ --cov-report term --cov-config=.coveragerc --cov=eva/ -s -v --log-level=WARNING ${1:-} -m "benchmark"
-test_code=$?
-if [ $test_code -ne 0 ];
+if [ -e ".benchmarks" ];
 then
-    exit $test_code
+    echo "SUBSEQUENT RUN"
+    # SUBSEQUENT RUNS
+    PYTHONPATH=./ pytest test/  --benchmark-autosave --benchmark-compare  -s -v --benchmark-compare-fail=min:50% --log-level=WARNING ${1:-} -m "benchmark"
+    test_code=$?
+    if [ $test_code -ne 0 ];
+    then
+        exit $test_code
+    fi
+else
+    echo "FIRST RUN"
+    # FIRST RUN FOR REFERENCE
+    PYTHONPATH=./ pytest test/  --benchmark-autosave -s -v --log-level=WARNING ${1:-} -m "benchmark"
+    test_code=$?
+    if [ $test_code -ne 0 ];
+    then
+        exit $test_code
+    fi
 fi
 
 # restore __init__.py if it exists

test/benchmark_tests/test_benchmark_pytorch.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import pytest
 
 from eva.server.command_handler import execute_query_fetch_all
@@ -48,6 +49,7 @@ def test_should_run_pytorch_and_facenet(benchmark, setup_pytorch_tests):
 
     select_query = """SELECT FaceDetector(data) FROM MyVideo
                     WHERE id < 5;"""
+
     actual_batch = benchmark(execute_query_fetch_all, select_query)
     assert len(actual_batch) == 5
 

test/integration_tests/test_optimizer_rules.py

@@ -15,6 +15,8 @@
 import unittest
 from test.util import load_inbuilt_udfs
 
+from mock import patch
+
 from eva.catalog.catalog_manager import CatalogManager
 from eva.configuration.constants import EVA_ROOT_DIR
 from eva.optimizer.plan_generator import PlanGenerator
@@ -40,17 +42,20 @@ def setUpClass(cls):
     def tearDownClass(cls):
         execute_query_fetch_all("DROP TABLE IF EXISTS MyVideo;")
 
-    def test_should_benefit_from_pushdown(self):
+    @patch("eva.expression.function_expression.FunctionExpression.evaluate")
+    def test_should_benefit_from_pushdown(self, evaluate_mock):
         query = """SELECT id, obj.labels
                   FROM MyVideo JOIN LATERAL
-                    YoloV5(data) AS obj(labels, bboxes, scores)
+                    FastRCNNObjectDetector(data) AS obj(labels, bboxes, scores)
                   WHERE id < 2;"""
 
         time_with_rule = Timer()
         result_with_rule = None
         with time_with_rule:
             result_with_rule = execute_query_fetch_all(query)
 
+        evaluate_count_with_rule = evaluate_mock.call_count
+
         time_without_rule = Timer()
         result_without_pushdown_rules = None
         with time_without_rule:
@@ -64,11 +69,14 @@ def test_should_benefit_from_pushdown(self):
 
         self.assertEqual(result_without_pushdown_rules, result_with_rule)
 
-        # without rule it should be slow as we end up running yolo on all the frames
-        self.assertGreater(
-            time_without_rule.total_elapsed_time, 3 * time_with_rule.total_elapsed_time
+        evaluate_count_without_rule = (
+            evaluate_mock.call_count - evaluate_count_with_rule
         )
 
+        # without rule it should be slow as we end up running the function
+        # on all the frames
+        self.assertGreater(evaluate_count_without_rule, 3 * evaluate_count_with_rule)
+
         result_without_xform_rule = None
         with disable_rules([XformLateralJoinToLinearFlow()]) as rules_manager:
             custom_plan_generator = PlanGenerator(rules_manager)
@@ -81,7 +89,7 @@ def test_should_benefit_from_pushdown(self):
     def test_should_pushdown_without_pushdown_join_rule(self):
         query = """SELECT id, obj.labels
                   FROM MyVideo JOIN LATERAL
-                    YoloV5(data) AS obj(labels, bboxes, scores)
+                    FastRCNNObjectDetector(data) AS obj(labels, bboxes, scores)
                   WHERE id < 2;"""
 
         time_with_rule = Timer()

test/integration_tests/test_pytorch.py

@@ -62,43 +62,27 @@ def tearDownClass(cls):
         execute_query_fetch_all("DROP TABLE IF EXISTS MemeImages;")
 
     @pytest.mark.torchtest
-    def test_should_run_pytorch_and_fastrcnn(self):
-        select_query = """SELECT FastRCNNObjectDetector(data) FROM MyVideo
-                        WHERE id < 5;"""
-        actual_batch = execute_query_fetch_all(select_query)
-        self.assertEqual(len(actual_batch), 5)
-
-    @pytest.mark.torchtest
-    def test_should_run_pytorch_and_yolo(self):
-        select_query = """SELECT YoloV5(data) FROM MyVideo
-                        WHERE id < 5;"""
-        actual_batch = execute_query_fetch_all(select_query)
-        self.assertEqual(len(actual_batch), 5)
-
-    @pytest.mark.torchtest
-    def test_should_run_pytorch_and_mvit(self):
-
-        execute_query_fetch_all(Mvit_udf_query)
-        select_query = """SELECT FIRST(id), MVITActionRecognition(SEGMENT(data)) FROM Actions
-                       GROUP BY '16f';"""
+    def test_should_run_pytorch_and_fastrcnn_with_lateral_join(self):
+        select_query = """SELECT id, obj.labels
+                          FROM MyVideo JOIN LATERAL
+                          FastRCNNObjectDetector(data)
+                          AS obj(labels, bboxes, scores)
+                         WHERE id < 2;"""
         actual_batch = execute_query_fetch_all(select_query)
-        self.assertEqual(len(actual_batch), 9)
-        res = actual_batch.frames
-        # TODO ACTION: Test case for aliases
-        for idx in res.index:
-            self.assertTrue("yoga" in res["mvitactionrecognition.labels"][idx])
+        self.assertEqual(len(actual_batch), 2)
 
     @pytest.mark.torchtest
-    def test_should_run_pytorch_and_fastrcnn_and_mvit(self):
+    def test_should_run_pytorch_and_yolo_and_mvit(self):
         execute_query_fetch_all(Mvit_udf_query)
 
         select_query = """SELECT FIRST(id),
-                                 YoloV5(FIRST(data)),
-                                 MVITActionRecognition(SEGMENT(data))
-                       FROM Actions
-                       GROUP BY '16f';"""
+                            YoloV5(FIRST(data)),
+                            MVITActionRecognition(SEGMENT(data))
+                            FROM Actions
+                            WHERE id < 32
+                            GROUP BY '16f'; """
         actual_batch = execute_query_fetch_all(select_query)
-        self.assertEqual(len(actual_batch), 9)
+        self.assertEqual(len(actual_batch), 2)
 
         res = actual_batch.frames
         for idx in res.index:

test/integration_tests/test_show_info_executor.py

@@ -12,9 +12,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
+import sys
 import unittest
 
 import pandas as pd
+import pytest
 
 from eva.catalog.catalog_manager import CatalogManager
 from eva.configuration.constants import EVA_ROOT_DIR
@@ -73,7 +76,19 @@ def test_show_udfs(self):
         self.assertTrue(all(expected_df.name == result.frames.name))
         self.assertTrue(all(expected_df.type == result.frames.type))
 
+    @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows")
     def test_show_tables(self):
+
+        result = execute_query_fetch_all("SHOW TABLES;")
+        self.assertEqual(len(result), 3)
+        expected = {"name": ["MyVideo", "MNIST", "Actions"]}
+        expected_df = pd.DataFrame(expected)
+        self.assertEqual(result, Batch(expected_df))
+
+        # Stop and restart server
+        os.system("nohup eva_server --stop")
+        os.system("nohup eva_server --start &")
+
         result = execute_query_fetch_all("SHOW TABLES;")
         self.assertEqual(len(result), 3)
         expected = {"name": ["MyVideo", "MNIST", "Actions"]}