chore: Adding pip-compile-multi et al. (apache#10499)

* chore: Adding pip-compile-multi et al * Specify requirements.txt path for fossa * [ci] Fixing CI Co-authored-by: John Bodley <[email protected]> Co-authored-by: Jesse Yang <[email protected]>
tkopczuk · Aug 5, 2020 · cda764f · cda764f
1 parent 72ced53
commit cda764f
Show file tree

Hide file tree

Showing 41 changed files with 480 additions and 302 deletions.
diff --git a/.fossa.yml b/.fossa.yml
@@ -32,7 +32,11 @@ analyze:
     type: pip
     target: docs
     path: docs
+    options:
+      requirements: ./requirements/documentation.txt
   - name: .
     type: pip
     target: .
     path: .
+    options:
+      requirements: ./requirements/testing.txt
diff --git a/.github/workflows/bashlib.sh b/.github/workflows/bashlib.sh
@@ -33,24 +33,21 @@ say() {
 
 # default command to run when the `run` input is empty
 default-setup-command() {
-  pip-install
+  apt-get-install
+  pip-upgrade
 }
 
-# install python dependencies
-pip-install() {
-  cd "$GITHUB_WORKSPACE"
-
-  # Pip cache saves at most about 20s on a good day
-  # cache-restore pip
+apt-get-install() {
+    say "::group::apt-get install dependencies"
+    sudo apt-get update && sudo apt-get install --yes \
+      libsasl2-dev
+    say "::endgroup::"
+}
 
-  say "::group::Install Python pacakges"
+pip-upgrade() {
+  say "::group::Upgrade pip"
   pip install --upgrade pip
-  pip install -r requirements.txt
-  pip install -r requirements-dev.txt
-  pip install -e ".[postgres,mysql]"
   say "::endgroup::"
-
-  # cache-save pip
 }
 
 # prepare (lint and build) frontend code
@@ -123,6 +120,7 @@ testdata() {
   say "::group::Load test data"
   # must specify PYTHONPATH to make `tests.superset_test_config` importable
   export PYTHONPATH="$GITHUB_WORKSPACE"
+  pip install -e .
   superset db upgrade
   superset load_test_users
   superset load_examples --load-test-data

diff --git a/.github/workflows/caches.js b/.github/workflows/caches.js
@@ -36,7 +36,7 @@ const assetsConfig = {
 module.exports = {
   pip: {
     path: [`${homeDirectory}/.cache/pip`],
-    hashFiles: [`${workspaceDirectory}/requirements*.txt`],
+    hashFiles: [`${workspaceDirectory}/requirements/*.txt`],
   },
   npm: {
     path: [`${homeDirectory}/.npm`],

diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml
@@ -33,7 +33,7 @@ jobs:
 
         EOF
 
-        if [[ "${FILES}" =~ (.*package*\.json|requirements.*\.txt|setup\.py) ]]; then
+        if [[ "${FILES}" =~ (.*package*\.json|requirements/*\.txt|setup\.py) ]]; then
           echo "Detected dependency changes... running fossa check"
 
           ./scripts/fossa.sh

diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml
@@ -37,15 +37,18 @@ jobs:
       uses: actions/[email protected]
       with:
         python-version: '3.6'
-
     - name: Install dependencies
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          npm-install && build-instrumented-assets
-          pip-install && setup-postgres && testdata
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
+          setup-postgres
+          testdata
+          npm-install
+          build-instrumented-assets
           cypress-install
-
     - name: Run Cypress
       uses: apache-superset/cached-dependencies@b90713b
       env:

diff --git a/.github/workflows/superset-python.yml b/.github/workflows/superset-python.yml
@@ -8,8 +8,6 @@ jobs:
     strategy:
       matrix:
         python-version: [3.6]
-    env:
-      PYTHON_LINT_TARGET: setup.py superset tests
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
@@ -19,16 +17,37 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       uses: apache-superset/cached-dependencies@b90713b
-    - name: black
-      run: black --check $(echo $PYTHON_LINT_TARGET)
-    - name: mypy
-      run: mypy $(echo $PYTHON_LINT_TARGET)
-    - name: isort
-      run: isort --check-only --recursive $(echo $PYTHON_LINT_TARGET)
+      with:
+        run: |
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
     - name: pylint
       # `-j 0` run Pylint in parallel
       run: pylint -j 0 superset
 
+  pre-commit:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [3.6]
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/[email protected]
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      uses: apache-superset/cached-dependencies@b90713b
+      with:
+        run: |
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/integration.txt
+    - name: pre-commit
+      run: pre-commit run --all-files
+
   docs:
     runs-on: ubuntu-18.04
     strategy:
@@ -45,8 +64,9 @@ jobs:
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          pip-install
-          pip install -r docs/requirements.txt
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/documentation.txt
     - name: Build documentation
       run: sphinx-build -b html docs _build/html -W
 
@@ -66,8 +86,9 @@ jobs:
         uses: apache-superset/cached-dependencies@b90713b
         with:
           run: |
-            pip-install
-            pip install -r docs/requirements.txt
+            apt-get-install
+            pip-upgrade
+            pip install -r requirements/documentation.txt
       - name: Test babel extraction
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct
 
@@ -107,7 +128,9 @@ jobs:
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          pip-install
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
           setup-postgres
     - name: Run celery
       run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
@@ -151,7 +174,9 @@ jobs:
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          pip-install
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
           setup-mysql
     - name: Run celery
       run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
@@ -188,7 +213,9 @@ jobs:
       uses: apache-superset/cached-dependencies@b90713b
       with:
         run: |
-          pip-install
+          apt-get-install
+          pip-upgrade
+          pip install -r requirements/testing.txt
           mkdir ${{ github.workspace }}/.temp
     - name: Run celery
       run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -15,29 +15,35 @@
 # limitations under the License.
 #
 repos:
--   repo: https://github.com/ambv/black
-    rev: 19.10b0
-    hooks:
-    -   id: black
-        language_version: python3
-
--   repo: https://github.com/asottile/seed-isort-config
+  - repo: https://github.com/asottile/seed-isort-config
     rev: v1.9.3
     hooks:
-    -   id: seed-isort-config
-
--   repo: https://github.com/pre-commit/mirrors-isort
+      - id: seed-isort-config
+  - repo: https://github.com/pre-commit/mirrors-isort
     rev: v4.3.21
     hooks:
-    -   id: isort
-
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.2.3
+      - id: isort
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.770
+    hooks:
+      - id: mypy
+  - repo: https://github.com/peterdemin/pip-compile-multi
+    rev: v1.5.8
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: check-docstring-first
-    -   id: check-added-large-files
-    -   id: check-yaml
+      - id: pip-compile-multi-verify
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-docstring-first
+      - id: check-yaml
         exclude: ^helm/superset/templates/
-    -   id: debug-statements
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+        args: ["--markdown-linebreak-ext=md"]
+  - repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+      - id: black
+        language_version: python3
diff --git a/.pylintrc b/.pylintrc
@@ -25,7 +25,7 @@
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=CVS
+ignore=CVS,migrations
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.

diff --git a/.rat-excludes b/.rat-excludes
@@ -1,3 +1,4 @@
+# Note: these patterns are applied to single files or directories, not full paths
 .gitignore
 .gitattributes
 .gitkeep
@@ -40,7 +41,7 @@ _static/*
 .buildinfo
 searchindex.js
 # auto generated
-requirements.txt
+requirements/*
 # vendorized
 vendor/*
 # github configuration

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -307,7 +307,7 @@ Finally, to make changes to the rst files and build the docs using Sphinx,
 you'll need to install a handful of dependencies from the repo you cloned:
 
 ```bash
-pip install -r docs/requirements.txt
+pip install -r requirements/documentation.txt
 ```
 
 To get the feel for how to edit and build the docs, let's edit a file, build
@@ -375,7 +375,7 @@ For example, the image referenced above actually lives in `superset-frontend/ima
 Generate the API documentation with:
 
 ```bash
-pip install -r docs/requirements.txt
+pip install -r requirements/documentation.txt
 python setup.py build_sphinx
 ```
 
@@ -399,8 +399,7 @@ virtualenv -p python3 venv # setup a python3.6 virtualenv
 source venv/bin/activate
 
 # Install external dependencies
-pip install -r requirements.txt
-pip install -r requirements-dev.txt
+pip install -r requirements/local.txt
 
 # Install Superset in editable (development) mode
 pip install -e .
@@ -553,7 +552,7 @@ in combined feature flags of `{ 'FOO': True, 'BAR': True, 'BAZ': True }`.
 Superset uses Git pre-commit hooks courtesy of [pre-commit](https://pre-commit.com/). To install run the following:
 
 ```bash
-pip3 install -r requirements-dev.txt
+pip3 install -r requirements/integration.txt
 pre-commit install
 ```
 

diff --git a/Dockerfile b/Dockerfile
@@ -31,9 +31,9 @@ RUN mkdir /app \
 
 # First, we just wanna install requirements, which will allow us to utilize the cache
 # in order to only build if and only if requirements change
-COPY ./requirements.txt /app/
+COPY ./requirements/*.txt /app/
 RUN cd /app \
-        && pip install --no-cache -r requirements.txt
+        && pip install --no-cache -r requirements/local.txt
 
 
 ######################################################################
@@ -114,14 +114,13 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 ######################################################################
 FROM lean AS dev
 
-COPY ./requirements* ./docker/requirements* /app/
+COPY ./requirements/*.txt ./docker/requirements/ /app/
 
 USER root
 # Cache everything for dev purposes...
 RUN cd /app \
     && pip install --ignore-installed -e . \
-    && pip install --ignore-installed -r requirements.txt \
-    && pip install --ignore-installed -r requirements-dev.txt \
+    && pip install --ignore-installed -r requirements/local.txt \
     && pip install --ignore-installed -r requirements-extra.txt \
     && pip install --ignore-installed -r requirements-local.txt || true
 USER superset
diff --git a/Dockerfile-dev b/Dockerfile-dev
@@ -20,12 +20,12 @@
 ######################################################################
 FROM preset/superset:dev
 
-COPY ./requirements* ./docker/requirements* /app/
+COPY ./requirements/*.txt ./docker/requirements* /app/
 
 USER root
 RUN cd /app \
     && pip install -e . \
-    && pip install --no-cache -r requirements.txt -r requirements-dev.txt \
+    && pip install --no-cache -r requirements/local.txt \
     && pip install --no-cache -r requirements-extra.txt \
     && pip install --no-cache -r requirements-local.txt || true
 USER superset
diff --git a/RELEASING/Dockerfile.make_docs b/RELEASING/Dockerfile.make_docs
@@ -20,8 +20,7 @@ ARG VERSION
 RUN git clone --depth 1 --branch ${VERSION} https://github.com/apache/incubator-superset.git /superset
 WORKDIR /superset
 # install doc dependencies
-RUN pip install -r requirements.txt \
-    && pip install -r docs/requirements.txt
+RUN pip install -r requirements/documentation.txt
 # build the docs
 RUN python setup.py build_sphinx
 

diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py
@@ -86,8 +86,8 @@ class CeleryConfig(object):
 # the PYTHONPATH) in order to allow for local settings to be overridden
 #
 try:
-    from superset_config_docker import *  # noqa
     import superset_config_docker
+    from superset_config_docker import *  # noqa
 
     logger.info(
         f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]"