migrate to setuptools (HEPCloud#264)

Also move flake8 to a github action

RB: https://fermicloud140.fnal.gov/reviews/r/329/

.github/actions/unittest-in-sl7-docker/action.yaml → .github/actions/pytest-in-sl7-container/action.yaml

@@ -1,5 +1,5 @@
 name: 'Unit Tests'
-description: 'Run unit tests in preconfigured SL7 docker image'
+description: 'Run pytest in preconfigured SL7 container image'
 runs:
   using: 'docker'
   image: 'Dockerfile'

.github/actions/pytest-in-sl7-container/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -x
+GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-`pwd`}
+export PATH=$PATH:/usr/pgsql-11/bin
+python3 -m pip install --upgrade pip --user
+python3 -m pip install --upgrade setuptools wheel setuptools-scm[toml] --user
+python3 -m pip install -r ${GITHUB_WORKSPACE}/requirements/requirements-runtime.txt --user
+python3 -m pip install -r ${GITHUB_WORKSPACE}/requirements/requirements-develop.txt --user
+python3 setup.py bdist_wheel
+# cannot use `tee` as it eats $? and we lose success/failure
+python3 -m pytest 2>&1 > pytest.log 
+RC=$?
+cat pytest.log
+exit ${RC}

.github/actions/rpmbuild-in-sl7-container/action.yaml

@@ -0,0 +1,5 @@
+name: 'RPM build'
+description: 'Run rpm build in preconfigured SL7 container image'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/rpmbuild-in-sl7-container/entrypoint.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -xe
+GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-`pwd`}
+python3 -m pip install --upgrade pip --user
+python3 -m pip install --upgrade setuptools wheel setuptools-scm[toml] --user
+python3 -m pip install -r ${GITHUB_WORKSPACE}/requirements/requirements-develop.txt --user
+python3 setup.py bdist_rpm

.github/workflows/ci.yaml

@@ -32,6 +32,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
           architecture: x64
 
+      - name: Merge requirements files into one
+        run: cat requirements/*.txt | sort -u | grep -v requirements-runtime.txt > requirements.txt
+
       - name: Cache pip
         uses: actions/cache@v2
         with:
@@ -52,17 +55,20 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade wheel
-          python -m pip install --upgrade pytest
-          python -m pip install -r requirements.txt
+          python3 -m pip install --upgrade pip
+          python3 -m pip install --upgrade wheel
+          python3 -m pip install --upgrade pytest
+          python3 -m pip install -r requirements.txt
+
+      - name: Ensure version.py exists
+        run: python3 setup.py bdist_wheel
 
       - name: Run Unit Tests on Python ${{ matrix.python-version }}
         env:
-          PYTHONPATH: ${{ github.workspace }}/../
           PYTEST_TIMEOUT: 90
+          PYTHONPATH: ${{ github.workspace }}/src
         run: |
-          pytest --cov-report term --cov-report xml
+          pytest --cov-report term --cov-report xml --cov=decisionengine --no-cov-on-fail
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
@@ -84,6 +90,9 @@ jobs:
           python-version: '3.9'
           architecture: 'x64'
 
+      - name: Merge requirements files into one
+        run: cat requirements/*.txt | sort -u | grep -v requirements-runtime.txt > requirements.txt
+
       - name: Cache pip
         uses: actions/cache@v2
         with:
@@ -104,16 +113,78 @@ jobs:
           python3 -m pip cache dir
 
       - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install --upgrade wheel
+          python3 -m pip install --upgrade pytest
+          python3 -m pip install -r requirements.txt
+
+      - name: Install sphinx expected fonts within the container
         run: |
           sudo apt-get update
-          sudo apt-get install -y python3-pip python3-psycopg2 latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended
-          python -m pip install --upgrade pip
-          python -m pip install -r requirements.txt
-          python -m pip install sphinx
+          sudo apt-get install -y latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended
+          sudo pip install -U pip
+          sudo pip3 install sphinx
+
+      - name: Ensure version.py exists
+        run: python3 setup.py bdist_wheel
 
       - name: Build documentation
         env:
-          PYTHONPATH: ${{ github.workspace }}/../
+          PYTHONPATH: ${{ github.workspace }}/src
         run: |
           cd doc
           make rst html latexpdf
+
+  run_flake8:
+    name: Run pytest-flake8 against code tree
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - name: checkout code tree
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+
+      - name: Merge requirements files into one
+        run: cat requirements/*.txt | sort -u | grep -v requirements-runtime.txt > requirements.txt
+
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}-3.9
+          restore-keys: |
+            ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}-
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Note Python version/implementation
+        run: |
+          which python3
+          python3 -c 'import sys; print(sys.version)'
+          python3 -c 'import platform; print(platform.python_implementation())'
+          python3 -m pip cache dir
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install --upgrade wheel
+          python3 -m pip install --upgrade pytest
+          python3 -m pip install -r requirements.txt
+
+      - name: Ensure version.py exists
+        run: python3 setup.py bdist_wheel
+
+      - name: Run Flake8 Tests on Python ${{ matrix.python-version }}
+        env:
+          PYTHONPATH: ${{ github.workspace }}/src
+        run: |
+          pytest -m flake8 --flake8

.github/workflows/gh-pages.yaml

@@ -22,6 +22,9 @@ jobs:
         python-version: '3.9'
         architecture: 'x64'
 
+    - name: Merge requirements files into one
+      run: cat requirements/*.txt | sort -u | grep -v requirements-runtime.txt > requirements.txt
+
     - name: Cache pip
       uses: actions/cache@v2
       with:
@@ -50,23 +53,17 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y latexmk texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended
-        sudo pip install -U pip
-        sudo pip3 install sphinx
+        sudo python3 -m pip install -U pip
+        sudo python3 -m pip install sphinx
 
     - name: Install required modules for AST to parse the codebase
-      run: pip3 install -r requirements.txt
+      run: python3 -m pip install -r requirements.txt
 
     - name: Setup safe/unique tmp dir for out of tree storage
       run: rm -rf /tmp/${{ github.sha }}
 
-    - name: Make source code importable
-      run: |
-        touch __init__.py
-        cd ..
-        touch __init__.py
-
-    - name: Set PYTHONPATH
-      run: echo "PYTHONPATH=${GITHUB_WORKSPACE}/../" >> $GITHUB_ENV
+    - name: Ensure version.py exists
+      run: python3 setup.py bdist_wheel
 
     - name: Run sphinx to make docs
       run: |

.github/workflows/pytest-in-sl7.yaml

@@ -0,0 +1,32 @@
+---
+name: Test in EL7 container
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - '**'          # matches every branch
+      - '!gh-pages'   # excludes gh-pages branch
+
+jobs:
+  rpm_build_job:
+    runs-on: ubuntu-latest
+    name: A job to test the decision engine framework on EL7
+
+    steps:
+      - name: checkout code tree
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Run the tests in a container (SL7/HEPCloud-CI)
+        uses: ./.github/actions/pytest-in-sl7-container
+
+      - name: Archive logs
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-log
+          path: pytest.log
+          if-no-files-found: error