Merge dev - new structure (EthTx#97)

- new app structure (more extendible)
- new Docker structure
- add scripts
- update `pre-commit`
- add `meinheld` worker
- add gunicorn conf
- fix doc

.env_sample

@@ -15,8 +15,6 @@ ETHERSCAN_KEY=
 # used in decoding process. But, it's not neccessary for running, If you don't want to use permanent
 # db or setup mongo, leave those values, mongomock package is used to simulate in-memory mongo.
 MONGO_CONNECTION_STRING=mongomock://localhost/ethtx
-MONGODB_DB=ethtx
-
 
 # Optional. Credentials for accessing semantics editor page, available under '/semantics/<str:address>'
 ETHTX_ADMIN_USERNAME=admin

.github/workflows/aws-prod.yml

@@ -73,7 +73,7 @@ jobs:
         echo "-----------------------"
         echo "Build step 2/3: Testing builded image"
         echo "-----------------------"
-        docker run  $ECR_REGISTRY/$ECR_REPO_NAME:$SHA_TAG make test
+        docker run  $ECR_REGISTRY/$ECR_REPO_NAME:$SHA_TAG pipenv run python -m pytest .
         echo "-----------------------"
         echo "Build step 3/3: Pushing builded image to $ECR_REPO_NAME"
         echo "-----------------------"

.github/workflows/aws-staging.yml

@@ -73,7 +73,7 @@ jobs:
         echo "-----------------------"
         echo "Build step 2/3: Testing builded image"
         echo "-----------------------"
-        docker run  $ECR_REGISTRY/$ECR_REPO_NAME:$SHA_TAG make test
+        docker run  $ECR_REGISTRY/$ECR_REPO_NAME:$SHA_TAG pipenv run python -m pytest .
         echo "-----------------------"
         echo "Build step 3/3: Pushing builded image to $ECR_REPO_NAME"
         echo "-----------------------"

.pre-commit-config.yaml

@@ -3,9 +3,18 @@ repos:
     rev: 21.12b0
     hooks:
       - id: black
-        language_version: python3.8
-        name: Backend:black
-        alias: backend-black
+        language_version: python3.9
+        name: EthTx_ce:black
+        alias: ethtx_ce-black
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        language_version: python3.9
+        name: EthTx_ce:flake8
+        alias: ethtx_ce-flake8
+        args: [ --config=ethtx_ce/.flake8 ]
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
@@ -18,17 +27,9 @@ repos:
   - repo: local
     hooks:
       - id: pytest
+        files: ./ethtx_ce/tests/
         name: pytest
         language: system
         entry: make test
         pass_filenames: false
-        always_run: true
-
-#  - repo: https://gitlab.com/pycqa/flake8
-#    rev: 3.9.2
-#    hooks:
-#      - id: flake8
-#        language_version: python3.8
-#        name: Backend:flake8
-#        alias: backend-flake8
-#        args: [--config=.flake8]
+        always_run: true

DEVELOPMENT.md

@@ -1,13 +1,17 @@
 # Local Development
-This repository contains 2 basic applications: `frontend` & `api`.
-It is easy to manage, and you can easily add new local application(s).
+
+This repository contains 2 basic applications: `frontend` & `api`. It is easy to manage, and you can easily add new
+local application(s).
 
 ## Basic structure
+
 Application is based on [blueprints](https://flask.palletsprojects.com/en/2.0.x/blueprints/).
 
 New extension requires:
- - new Python Package in ![ethtx_ce](ethtx_ce) subdirectory.
-- `create_app` function (created in new package in `init` file) which returns `Flask` object by calling ![app factory](ethtx_ce/factory.py) file.
+
+- new Python Package in ![ethtx_ce](ethtx_ce/app) subdirectory.
+- `create_app` function (created in new package in `init` file) which returns `Flask` object by
+  calling ![app factory](ethtx_ce/app/factory.py) file.
 - calling a function above in a `wsgi.py` file with assigned url prefix.
 
 These simple steps allow you to add new extension and integrate with entire application.

Dockerfile

@@ -1,11 +1,27 @@
 FROM python:3.9
 
-ENV PYTHONDONTWRITEBYTECODE 1
+WORKDIR /app/
 
-RUN mkdir /app
-WORKDIR /app
-ADD . /app
-COPY Pip* /app/
+# Upgrade pip, install pipenv
+RUN pip install --upgrade pip && pip install pipenv
+
+# Copy Pipfile* in case it doesn't exist in the repo
+COPY Pipfile* /app/
+
+COPY ./ethtx_ce/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+COPY ./ethtx_ce/start.sh  /start.sh
+RUN chmod +x /start.sh
+
+COPY ./ethtx_ce/start-reload.sh /start-reload.sh
+RUN chmod +x /start-reload.sh
+
+COPY ./ethtx_ce/gunicorn_conf.py /gunicorn_conf.py
+
+COPY Makefile /Makefile
+
+RUN bash -c "pipenv install --dev --deploy"
 
 ARG GIT_URL
 ENV GIT_URL=$GIT_URL
@@ -14,10 +30,10 @@ ARG GIT_SHA
 ENV GIT_SHA=$GIT_SHA
 
 ARG CI=1
-RUN pip install --upgrade pip && \
-    pip install pipenv && \
-    pipenv install --dev --deploy
+
+COPY ./ethtx_ce /app
+ENV PYTHONPATH=/app
 
 EXPOSE 5000
 
-CMD cd /app && pipenv run gunicorn --workers 4 --max-requests 4000 --timeout 600 --bind :5000 wsgi:app
+ENTRYPOINT ["/entrypoint.sh"]

Makefile

@@ -1,5 +1,3 @@
-
-
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
@@ -13,24 +11,24 @@ run-database:  ## Run only a local database required for local development
 	docker-compose up -d mongo mongo-express
 
 run-local:
-	FLASK_APP=wsgi.py FLASK_DEBUG=1 pipenv run flask run --host=0.0.0.0 --port 5000
+	PYTHONPATH=./ethtx_ce FLASK_APP=ethtx_ce/app/wsgi.py FLASK_DEBUG=1 pipenv run flask run --host=0.0.0.0 --port 5000
 
 run-prod:
 	fuser -k 5000/tcp || true
-	pipenv run gunicorn --workers 4 --max-requests 4000 --timeout 600 --bind :5000 wsgi:app
+	PYTHONPATH=./ethtx_ce pipenv run gunicorn --workers 4 --max-requests 4000 --timeout 600 --bind :5000 app.wsgi:app
 
 run-docker:
 	fuser -k 5000/tcp || true
 	docker-compose up -d
 
 run-test-docker:
-	docker run -it ethtx_ce make test
+	docker run -it ethtx_ce pipenv run python -m pytest .
 
 test:
-	PYTHONPATH=. pipenv run python -m pytest tests
+	PYTHONPATH=./ethtx_ce pipenv run python -m pytest ethtx_ce/tests/
 
 test-all:
-	PYTHONPATH=. pipenv run python -m pytest .
+	PYTHONPATH=./ethtx_ce pipenv run python -m pytest .
 
 setup:
 	pipenv install --dev

Pipfile

@@ -13,6 +13,7 @@ gunicorn = ">=20.1.0"
 flask-httpauth = ">=4.5.0"
 gitpython = ">=3.1.24"
 jsonpickle = ">=2.0.0"
+meinheld = ">=1.0.2"
 
 [dev-packages]
 black = "==21.12b0"

README.md

@@ -25,60 +25,78 @@
 ---
 
 # Description
-This project represents usage of [EthTx](https://github.com/ethtx/ethtx) decoding library in form of a 
-website. If you are looking for implementation of the said decoding functionalities, please refer to [EthTx](https://github.com/ethtx/ethtx) repository.
+
+This project represents usage of [EthTx](https://github.com/ethtx/ethtx) decoding library in form of a website. If you
+are looking for implementation of the said decoding functionalities, please refer
+to [EthTx](https://github.com/ethtx/ethtx) repository.
 
 # Local environment
 
 Here is a list of steps to recreate local environment on <b>Ubuntu</b> distribution.
 
-1. Install needed packages using `apt`
+1. Install needed packages using `apt`:
+
+    ```shell
+    apt install docker-compose python3-pip pipenv make
+    ```
+2. Run:
+
+    ```shell
+    pipenv install
+    ```
 
-      ```shell
-    apt install docker-compose python3-pip pipenv make 
-      ```
-2. Run `pipenv install`. This will create new `venv` with all required packages installed, including `ethtx` library
 3. Copy `.env_sample` to `.env` and fill required field according to description
-```
-  # Proper nodes are required to run ethtx, provide connection strings for chains which will be used.
-  MAINNET_NODE_URL=https://geth-erigon-node:8545
-  # KOVAN_NODE_URL=
-  # RINKEBY_NODE_URL=
-
-  # EthTx supports multiple nodes, if one is unavailable, it will use others. You only need to specify them with a comma
-  # Example: MAINNET_NODE_URL=https://geth-erigon-node:8545,https://geth1-erigon-node:8545
-
-
-  # Etherscan API is used to get contract source code, required for decoding process
-  # You can get free key here https://etherscan.io/apis
-  ETHERSCAN_KEY=
-
-  # Optional. Those represent data required for connecting to mongoDB. It's used for caching semantics
-  # used in decoding process. But, it's not neccessary for running, If you don't want to use permanent
-  # db or setup mongo, leave those values, mongomock package is used to simulate in-memory mongo.
-  MONGO_CONNECTION_STRING=mongomock://localhost
-  MONGODB_DB=ethtx
-
-
-  # Optional. Credentials for accessing semantics editor page, available under '/semantics/<str:address>'
-  ETHTX_ADMIN_USERNAME=admin
-  ETHTX_ADMIN_PASSWORD=admin
-
-  # Optional. Api key used for exposing 
-  API_KEY=
-
-  # Optional. Valid values are ['production', 'staging', 'development']. Those mainly
-  # dictate what options are used for flask debugging and logging
-  ENV=development
-```
-4. Run `pipenv run python3 wsgi.py` for `make run-local`. This will setup new server on host 0.0.0.0 port 5000.
-5. Now `ethtx_ce` should be accessible through link [http://localhost:5000](http://localhost:5000) 
 
+    ```
+      # Proper nodes are required to run ethtx, provide connection strings for chains which will be used.
+      MAINNET_NODE_URL=https://geth-erigon-node:8545
+      # KOVAN_NODE_URL=
+      # RINKEBY_NODE_URL=
+
+      # EthTx supports multiple nodes, if one is unavailable, it will use others. You only need to specify them with a comma
+      # Example: MAINNET_NODE_URL=https://geth-erigon-node:8545,https://geth1-erigon-node:8545
+
+
+      # Etherscan API is used to get contract source code, required for decoding process
+      # You can get free key here https://etherscan.io/apis
+      ETHERSCAN_KEY=
+
+      # Optional. Those represent data required for connecting to mongoDB. It's used for caching semantics
+      # used in decoding process. But, it's not neccessary for running, If you don't want to use permanent
+      # db or setup mongo, leave those values, mongomock package is used to simulate in-memory mongo.
+      MONGO_CONNECTION_STRING=mongomock://localhost
+      MONGODB_DB=ethtx
+
+
+      # Optional. Credentials for accessing semantics editor page, available under '/semantics/<str:address>'
+      ETHTX_ADMIN_USERNAME=admin
+      ETHTX_ADMIN_PASSWORD=admin
+
+      # Optional. Api key used for exposing
+      API_KEY=
+
+      # Optional. Valid values are ['production', 'staging', 'development']. Those mainly
+      # dictate what options are used for flask debugging and logging
+      ENV=development
+    ```
+
+4. Run
+    ```shell
+    PYTHONPATH=./ethtx_ce FLASK_APP=ethtx_ce/app/wsgi.py pipenv run flask run --host=0.0.0.0 --port 5000
+    ```
+   or
+    ```shell
+    make run-local
+    ```
+   This will setup new server on host 0.0.0.0 port 5000.
+5. Now `ethtx_ce` should be accessible through link [http://localhost:5000](http://localhost:5000)
 
 Use can also provided `docker-compose` for running this locally:
+
 ```shell
 docker-compose up
 ```
+
 Note, this also need proper `.env` file to function properly.
 
 # .env file
@@ -89,7 +107,6 @@ For proper functioning, `.env` file is required containing all database and 3rd
 Parameters `[CHAIN_ID]_NODE_URL` should hold valid urls to ethereum nodes; Parameter `ETHERSCAN_KEY` should be equal to
 Etherscan API key assigned to user.
 
-
 # API
 
 The EthTx APIs are provided as a community service and without warranty, so please use what you need and no more. We