diff --git a/README.md b/README.md index 1265d9094..c9cca1366 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Documentation for EVA is located [here](https://evagatech.readthedocs.io/). ### Installation Instructions for Developers -To install EVA from source, use a virtual environment and the pip package manager. EVA requires JAVA 8 for generating the parser. +To install EVA from source, use a virtual environment and the pip package manager. EVA requires JAVA 8 for generating the parser. Detailed development guide is available [here](https://evagatech.readthedocs.io/en/latest/source/contribute/contributing.html). ```shell git clone https://github.com/georgia-tech-db/eva.git && cd eva @@ -132,7 +132,7 @@ python3 -m venv env38 # to create a virtual envir pip install --upgrade pip sudo -E apt install -y openjdk-8-jdk openjdk-8-jre # to install JAVA sh script/antlr4/generate_parser.sh # to generate the EVA parser -pip install . +python -m pip install install -e .[dev] ```

diff --git a/api-docs/_toc.yml b/api-docs/_toc.yml index b5338d6e4..f0ea665fd 100644 --- a/api-docs/_toc.yml +++ b/api-docs/_toc.yml @@ -6,9 +6,9 @@ parts: - file: source/overview/index - file: source/overview/installation - - caption: Tutorial - chapters: - - file: source/tutorials/tutorial + # - caption: Tutorial + # chapters: + # - file: source/tutorials/tutorials - caption: Reference chapters: @@ -21,13 +21,14 @@ parts: - file: source/reference/udf title: Register UDFs - - - caption: Developer Guides chapters: - - file: source/contribute/contributing + - file: source/contribute/index + title: Contributing sections: - - file: source/contribute/new_operator - - file: source/contribute/new_ddl_command + # - file: source/contribute/new_operator + - file: source/contribute/new_command - file: source/contribute/debugging + - file: source/contribute/release - file: source/contribute/packaging + title: Packaging diff --git a/api-docs/source/build.md b/api-docs/source/build.md new file mode 100644 index 000000000..e69de29bb diff --git a/api-docs/source/contribute/contributing.rst b/api-docs/source/contribute/contributing.rst deleted file mode 100644 index 3b8003eb0..000000000 --- a/api-docs/source/contribute/contributing.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _guide-contributing: - -Contributing -============= -To file a bug or request a feature, please file a GitHub issue. Pull requests are welcome. - -See the `people page `_ for the full listing of contributors. - -Packaging New Version Of EVA -============= - -1. Generate EVA grammar files. -```shell -bash script/antlr4/generate_parser.sh -``` - -2. Bump up version number in `setup.cfg` along with any additional dependencies. - -3. Create a new build locally. -```shell -python -m build -``` - -4. Upload build to pypi using credentials. -```shell -python -m twine upload dist/* -``` diff --git a/api-docs/source/contribute/debugging.md b/api-docs/source/contribute/debugging.md new file mode 100644 index 000000000..a9226a391 --- /dev/null +++ b/api-docs/source/contribute/debugging.md @@ -0,0 +1,59 @@ +### Documentation on Debugging Eva: + +Using a debugger in the visual studio code, is one of the optimal ways of debugging your User Defined Function(UDF). This tutorial gives a detailed step-by-step process of using a debugger with Eva. This tutorial is python-specific. However, a similar debugging process can be followed for other languages and frameworks as well. + +## Pre-requisites: +Python should be installed and its environment should be defined in the VS Code. If not pre-installed, use the extensions section to install python and set-up the environment. Follow the [offical instructions](https://realpython.com/python-development-visual-studio-code). + + +## STEP 1: Installing and setting-up debugger on Visual Studio Code +---------------------------------------------------------------- + +The debug option in Visual Studio Code is as follows: + +The debug icon when pressed will give you the option to create a launch.json file. + + +While creating the JSON file, you will be prompted to select the environment to be used. Select the python environment from the command Palette at the top. If the python environment cannot be seen in the drop-down menu, try installing the python extension, and repeat the process. + +Once you select the python environment, a `launch.json` file will be created with the default configurations set to debug a simple .py file. + +More configurations can further be added to the file, to modify the environment variables or to debug an entire folder or workspace directory. + +Use the following approach for debugging the eva set-up: + +``` +{ + + "version": "0.2.0", + "configurations": [ + + { + "name": "Python: test_pytorch.py", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/test/integration_tests/test_pytorch.py", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}", + "env": {"PYTHONPATH": "${workspaceRoot}", + "PYSPARK_PYTHON" : "/nethome/username/miniconda/envs/eva/bin/python", + "PYSPARK_DRIVER_PYTHON": "/nethome/username/miniconda/envs/eva/bin/python"} + } + + ] +} +``` + + +You can modify the fields of the above JSON file as follows: + + `name`: It is the reader-friendly name to appear in the Debug launch configuration dropdown. + + `type`: The type of debugger to use for this launch configuration. + + `program`: The executable or file to run when launching the debugger. In the above example, test_integration.py will be executed by the debugger. + + `env`: Here you specify the environment variables. In the above example, the path for the conda environment for Eva has been specified. + + +Using these configurations, the debugger can be executed both locally as well as on the remote server. \ No newline at end of file diff --git a/api-docs/source/contribute/debugging.rst b/api-docs/source/contribute/debugging.rst deleted file mode 100644 index 1a42d50d1..000000000 --- a/api-docs/source/contribute/debugging.rst +++ /dev/null @@ -1,117 +0,0 @@ -.. _guide-debugging: - -Documentation on Debugging Eva: -=============================== - -Using a debugger in the visual studio code, is one of the optimal ways of debugging your User Defined Function(UDF). This tutorial gives a detailed step-by-step process of using a debugger with Eva. This tutorial is python-specific. However, a similar debugging process can be followed for other languages and frameworks as well. - -Pre-requisites: --------------- -Python should be installed and its environment should be defined in the VS Code. If not pre-installed, use the extensions section to install python and set-up the environment. -A detailed link for the same is as follows: - -https://realpython.com/python-development-visual-studio-code - - -STEP 1: Installing and setting-up debugger on Visual Studio Code ----------------------------------------------------------------- - -The debug option in Visual Studio Code is as follows: - -.. image:: images/run.png - -The debug icon when pressed will give you the option to create a launch.json file. - -.. image:: images/launch-configuration.png - -While creating the JSON file, you will be prompted to select the environment to be used. Select the python environment from the command Palette at the top. If the python environment can’t be seen in the drop-down menu, try installing the python extension, and repeat the process. - -.. image:: images/debug-environments.png - -Once you select the python environment, a launch.json file will be created with the default configurations set to debug a simple .py file. - -More configurations can further be added to the file, to modify the environment variables or to debug an entire folder or workspace directory. - -Use the following approach for debugging the eva set-up: - -{ - - "version": "0.2.0", - "configurations": [ - - { - "name": "Python: test_pytorch.py", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/test/integration_tests/test_pytorch.py", - "console": "integratedTerminal", - "cwd": "${workspaceFolder}", - "env": {"PYTHONPATH": "${workspaceRoot}", - "PYSPARK_PYTHON" : "/nethome/username/miniconda/envs/eva/bin/python", - "PYSPARK_DRIVER_PYTHON": "/nethome/username/miniconda/envs/eva/bin/python"} - } - - ] -} - - - -You can modify the fields of the above JSON file as follows: - - #. “name”: It is the reader-friendly name to appear in the Debug launch configuration dropdown. - - - #. “type”: The type of debugger to use for this launch configuration. - - - #. “program”: The executable or file to run when launching the debugger. In the above example, test_integration.py will be executed by the debugger. - - - #. “env”: Here you specify the environment variables. In the above example, the path for the conda environment for Eva has been specified. - - -Using these configurations, the debugger can be executed both locally as well as on the remote server. - -Step 2: Using the Debugger for Eva ------------------------------------ - -Before starting the debugger, ensure that the Eva Conda environment is activated. -Also, start the Eva server, by running python eva.py. - -To start using the debugger, set breakpoints in the file you want to debug. - -.. image:: images/breakpoints.png - -Now you can go to the debug menu, and you will see a play button at the top specifying the configurations you have mentioned in the launch.json file. - -Choose the appropriate configuration, and press the play button to start debugging. - -The debugger will automatically start running the Eva framework and will stop at the point where you have applied the breakpoint. - -Once the debugger stops at the breakpoint, consider the following: - -You can see the variables at each stage, the call stack as well as the list of breakpoints on the left-hand panel of the visual studio code. - -.. image:: images/debug-session.png - -Use the debug actions to start executing the program step-by-step from the breakpoint. For this purpose, the following debug actions have been provided: - - - #. Continue / Pause F5: To run your program starting from the breakpoint. - - - #. Step Over F10: To step over a specific function. - - - #. Step Into F11: To go inside a specific function and check its internal execution. - - - #. Step Out F11: To step out of a function. - - - #. Restart F5: To restart the debugging. - - - #. Stop F5: To stop the debugging. - -.. image:: images/toolbar.png diff --git a/api-docs/source/contribute/index.md b/api-docs/source/contribute/index.md new file mode 100644 index 000000000..a65fe3109 --- /dev/null +++ b/api-docs/source/contribute/index.md @@ -0,0 +1,72 @@ +### Contributing +We welcome all kinds of contributions to EVA. +- New features +- Code reviewing of PR +- Documentation +- Tutorials and Applications + +#### Setting up the development environment +To hack on EVA, you need to checkout the repository and build EVA from the source. +Follow the following instructions to build EVA and test your changes locally. +We recommend using a virtual environment and the pip package manager. EVA requires JAVA 8 for generating the parser. +``` +git clone https://github.com/georgia-tech-db/eva.git && cd eva +python3 -m venv env38 # to create a virtual environment +. env38/bin/activate +pip install --upgrade pip +sudo -E apt install -y openjdk-8-jdk openjdk-8-jre # to install JAVA +sh script/antlr4/generate_parser.sh # to generate the EVA parser +python -m pip install install -e .[dev] +``` + +#### Submitting a contribution +Follow the following steps to contribute to EVA: +* Merge the most recent changes from the master branch +``` + git remote add origin git@github.com:georgia-tech-db/eva.git + git pull . origin/master +``` +* Run the [test script](#testing) to ensure all the test cases pass. +* Run the `setup_git_hooks.sh` to add a git pre-push hook so that it runs the linter before pushing any changes. +* If you are adding a new SQL command, please add the example usage to the documentation. + +#### Testing + +Before merging the PR, the code must pass all the unit test cases. You can use the following script to run all the test cases locally. +``` +bash script/test/test.sh +``` +If you want to run a specific test file, use the following command. +``` +python -m pytest test/integration_tests/test_select_executor.py +``` +Use the following command to run a specific test case within a test file. +``` +python -m pytest test/integration_tests/test_select_executor.py -k 'test_should_load_and_select_in_table' +``` + +#### Code Style +We use the [black](https://github.com/psf/black) code style for formatting our python code. For docstrings and documentation, we use [Google pydoc format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). + +``` +def function_with_types_in_docstring(param1, param2) -> bool: + """Example function with types documented in the docstring. + + Additional explanatory text can be added in paragraphs. + + Args: + param1 (int): The first parameter. + param2 (str): The second parameter. + + Returns: + bool: The return value. True for success, False otherwise. + +``` + +##### Lint and Formatting +Before merging, the PR must pass the code formatting and linting test case. +On your local machine, run the following script to auto-format using `black` + +``` +python script/formatting/formatter.py +``` \ No newline at end of file diff --git a/api-docs/source/contribute/new_ddl_command.md b/api-docs/source/contribute/new_command.md similarity index 98% rename from api-docs/source/contribute/new_ddl_command.md rename to api-docs/source/contribute/new_command.md index cf01fe4ac..6465a0155 100644 --- a/api-docs/source/contribute/new_ddl_command.md +++ b/api-docs/source/contribute/new_command.md @@ -1,4 +1,4 @@ -# Extending a new DDL command to EVA +# Adding new command This document is the note of tracing implementation of CREATE command. We also list all the file that need to be modified or added to extend a new DDL command to EVA. We hope this tutorial will be helpful when future users trying to implement new command in EVA. ## command_handler An input query string is handled by **Parser**, **StatementTOPlanConvertor**, **PlanGenerator**, and **PlanExcutor**. So let's discuss each part separately. @@ -22,7 +22,7 @@ The parser firstly generate **syntax tree** from the input string, and then tans The first part of Parser is build from two ANTLR file. ### parser/evaql -* `evaql_lexer.g4` - add key words(eg. CREATE, TABLE) under **Common Keywords** +* `evaql_lexer.g4` - add keywords(eg. CREATE, TABLE) under **Common Keywords** * `evaql_parser.g4` * Add new grammar name(eg. createTable) under **ddlStatement** * Write a new grammar, for example: diff --git a/api-docs/source/contribute/release.md b/api-docs/source/contribute/release.md new file mode 100644 index 000000000..eb7e64437 --- /dev/null +++ b/api-docs/source/contribute/release.md @@ -0,0 +1,17 @@ +### Version Release +Packaging New Version of EVA + +Generate EVA grammar files. +```bash +bash script/antlr4/generate_parser.sh +``` +Bump up version number in `version.py` along with any additional dependencies. + +Create a new build locally. +```bash +python -m build +``` +Upload build to pypi using credentials. +```bash +python -m twine upload dist/* +``` diff --git a/script/formatting/formatter.py b/script/formatting/formatter.py index 872e5aa8c..08f274554 100755 --- a/script/formatting/formatter.py +++ b/script/formatting/formatter.py @@ -120,16 +120,17 @@ def is_tool(name): elif name is ISORT_BINARY: req_version = ISORT_VERSION_REQUIRED if which(name) is None: - sys.exit( + LOG.error( f"{name} is not installed. Install the python package with:" - " pip install {name}=={req_version}" + f"pip install {name}=={req_version}" ) + sys.exit(1) else: installed_version = pkg_resources.get_distribution(name).version if installed_version != req_version: LOG.warning( f"EVA uses {name} {req_version}. The installed version is" - " {installed_version} which can result in different results." + f" {installed_version} which can result in different results." ) @@ -137,7 +138,6 @@ def format_file(file_path, add_header, strip_header, format_code): abs_path = os.path.abspath(file_path) LOG.info(file_path) - with open(abs_path, "r+") as fd: file_data = fd.read() if add_header: @@ -175,8 +175,9 @@ def format_file(file_path, add_header, strip_header, format_code): # AUTOFLAKE autoflake_command = FLAKE_BINARY + " --max-line-length 88 " + file_path # LOG.info(autoflake_command) - os.system(autoflake_command) - + ret_val = os.system(autoflake_command) + if ret_val: + sys.exit(1) # END WITH fd.close() @@ -286,11 +287,12 @@ def format_dir(dir_path, add_header, strip_header, format_code): ) for file in files: - LOG.info("Stripping headers : " + file) - format_file(file, False, True, False) + if file != "script/formatting/formatter.py": + LOG.info("Stripping headers : " + file) + format_file(file, False, True, False) - LOG.info("Adding headers : " + file) - format_file(file, True, False, False) + LOG.info("Adding headers : " + file) + format_file(file, True, False, False) LOG.info("Formatting File : " + file) format_file(file, False, False, True) diff --git a/script/formatting/pre-push.sh b/script/formatting/pre-push.sh new file mode 100755 index 000000000..d99574b7e --- /dev/null +++ b/script/formatting/pre-push.sh @@ -0,0 +1,21 @@ +#!/bin/sh +echo "Linting code before commiting changes" +echo "Running script/formatting/formatter.py" +python script/formatting/formatter.py +exit_status=$? +if [ $exit_status -ne 0 ]; then + echo "Oops! linting failed." + echo "Please make sure 'python script/formatting/formatter.py' executes without error before committing." + echo "In case you want to force commit, use the '--no-verify' flag while commiting." + exit 1 +fi + +if git diff-index --quiet HEAD --; then + exit 0 +else + echo "Code was reformatted. Please verify and stage the changes." + echo "List of files updated." + git --no-pager diff --name-only + exit 1 +fi + diff --git a/setup_git_hooks.sh b/setup_git_hooks.sh new file mode 100644 index 000000000..6ac2dd7c0 --- /dev/null +++ b/setup_git_hooks.sh @@ -0,0 +1,3 @@ +#!/bin/bash +chmod +x script/formatting/pre-push.sh +ln -s "$PWD"/script/formatting/pre-push.sh .git/hooks/pre-push