[integration] Update IT to use latest beam SDK

I also removed `--use-feature=2020-resolver` from the dockerfile since the latest pip version ignores it (the new resolver is default now). When re-generating the job-reqs.txt, I did the following: 1. built the job’s image 2. ran the image interactively (`docker run -it —rm -v $(pwd):/usr/src/app $img`) 3. installed pip-tools==5.4.0 (5.5+ has a new layout that would make the diff difficult to read) 4. ran `pip-compile job-requirements.in` 5. stopped the container and ran the integration tests for the particular job via KLIO_TEST_DIR=<dir> tox -c integration/tox.ini
pomdtr · Jan 12, 2021 · be89ab2 · be89ab2
1 parent 57aacd1
commit be89ab2
Show file tree

Hide file tree

Showing 19 changed files with 301 additions and 276 deletions.
diff --git a/integration/audio-spectrograms/Dockerfile b/integration/audio-spectrograms/Dockerfile
@@ -1,4 +1,4 @@
-FROM dataflow.gcr.io/v1beta3/python38-fnapi:2.24.0
+FROM dataflow.gcr.io/v1beta3/python38-fnapi:2.27.0
 
 WORKDIR /usr/src/app
 RUN mkdir -p /usr/src/config
@@ -19,7 +19,7 @@ RUN pip install ./exec
 RUN pip install ./audio
 
 COPY job-requirements.txt job-requirements.txt
-RUN pip install -r job-requirements.txt --use-feature=2020-resolver
+RUN pip install -r job-requirements.txt
 
 COPY __init__.py \
      run.py \
@@ -31,4 +31,4 @@ COPY __init__.py \
 
 ARG KLIO_CONFIG=klio-job.yaml
 COPY $KLIO_CONFIG /usr/src/config/.effective-klio-job.yaml
-RUN pip install . --use-feature=2020-resolver
+RUN pip install .
diff --git a/integration/audio-spectrograms/job-requirements.in b/integration/audio-spectrograms/job-requirements.in
@@ -1,6 +1,4 @@
-apache-beam[gcp]==2.24.0
+apache-beam[gcp]==2.27.0
 librosa
 pytest
-google-cloud-storage < 1.30.0
-google-cloud-kms < 2.0.0
 # Note: klio packages are installed via docker
diff --git a/integration/audio-spectrograms/job-requirements.txt b/integration/audio-spectrograms/job-requirements.txt
@@ -4,84 +4,85 @@
 #
 #    pip-compile job-requirements.in
 #
-apache-beam[gcp]==2.24.0  # via -r job-requirements.in
+apache-beam[gcp]==2.27.0  # via -r job-requirements.in
 appdirs==1.4.4            # via pooch
-attrs==20.2.0             # via pytest
-audioread==2.1.8          # via librosa
+attrs==20.3.0             # via pytest
+audioread==2.1.9          # via librosa
 avro-python3==1.9.2.1     # via apache-beam
-cachetools==3.1.1         # via apache-beam, google-auth
-certifi==2020.6.20        # via requests
-cffi==1.14.2              # via soundfile
-chardet==3.0.4            # via requests
+cachetools==4.2.0         # via apache-beam, google-auth
+certifi==2020.12.5        # via requests
+cffi==1.14.4              # via google-crc32c, soundfile
+chardet==4.0.0            # via requests
 crcmod==1.7               # via apache-beam
 decorator==4.4.2          # via librosa
 dill==0.3.1.1             # via apache-beam
 docopt==0.6.2             # via hdfs
-fastavro==0.23.6          # via apache-beam
-fasteners==0.15           # via google-apitools
+fastavro==1.2.3           # via apache-beam
+fasteners==0.16           # via google-apitools
 future==0.18.2            # via apache-beam
-google-api-core[grpc,grpcgcp]==1.22.2  # via google-cloud-bigquery, google-cloud-bigtable, google-cloud-core, google-cloud-datastore, google-cloud-dlp, google-cloud-kms, google-cloud-language, google-cloud-pubsub, google-cloud-spanner, google-cloud-videointelligence, google-cloud-vision
+google-api-core[grpc,grpcgcp]==1.24.1  # via google-cloud-bigquery, google-cloud-bigtable, google-cloud-build, google-cloud-core, google-cloud-datastore, google-cloud-dlp, google-cloud-language, google-cloud-pubsub, google-cloud-spanner, google-cloud-videointelligence, google-cloud-vision
 google-apitools==0.5.31   # via apache-beam
-google-auth==1.21.2       # via apache-beam, google-api-core, google-cloud-storage
-google-cloud-bigquery==1.27.2  # via apache-beam
-google-cloud-bigtable==1.4.0  # via apache-beam
-google-cloud-core==1.4.1  # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore, google-cloud-spanner, google-cloud-storage
-google-cloud-datastore==1.15.0  # via apache-beam
+google-auth==1.24.0       # via apache-beam, google-api-core
+google-cloud-bigquery==1.28.0  # via apache-beam
+google-cloud-bigtable==1.6.1  # via apache-beam
+google-cloud-build==2.0.0  # via apache-beam
+google-cloud-core==1.5.0  # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore, google-cloud-spanner
+google-cloud-datastore==1.15.3  # via apache-beam
 google-cloud-dlp==1.0.0   # via apache-beam
-google-cloud-kms==1.4.0   # via -r job-requirements.in
 google-cloud-language==1.3.0  # via apache-beam
 google-cloud-pubsub==1.7.0  # via apache-beam
-google-cloud-spanner==1.19.0  # via apache-beam
-google-cloud-storage==1.29.0  # via -r job-requirements.in
-google-cloud-videointelligence==1.15.0  # via apache-beam
+google-cloud-spanner==1.19.1  # via apache-beam
+google-cloud-videointelligence==1.16.1  # via apache-beam
 google-cloud-vision==1.0.0  # via apache-beam
-google-resumable-media==0.5.1  # via google-cloud-bigquery, google-cloud-storage
+google-crc32c==1.1.0      # via google-resumable-media
+google-resumable-media==1.2.0  # via google-cloud-bigquery
 googleapis-common-protos[grpc]==1.52.0  # via google-api-core, grpc-google-iam-v1
-grpc-google-iam-v1==0.12.3  # via google-cloud-bigtable, google-cloud-kms, google-cloud-pubsub, google-cloud-spanner
+grpc-google-iam-v1==0.12.3  # via google-cloud-bigtable, google-cloud-pubsub, google-cloud-spanner
 grpcio-gcp==0.2.2         # via apache-beam, google-api-core
-grpcio==1.32.0            # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, grpcio-gcp
+grpcio==1.34.0            # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, grpcio-gcp
 hdfs==2.5.8               # via apache-beam
 httplib2==0.17.4          # via apache-beam, google-apitools, oauth2client
 idna==2.10                # via requests
-importlib-metadata==1.7.0  # via pluggy, pytest
-iniconfig==1.0.1          # via pytest
-joblib==0.16.0            # via librosa, scikit-learn
+iniconfig==1.1.1          # via pytest
+joblib==1.0.0             # via librosa, scikit-learn
+libcst==0.3.16            # via google-cloud-build
 librosa==0.8.0            # via -r job-requirements.in
-llvmlite==0.34.0          # via numba
+llvmlite==0.35.0          # via numba
 mock==2.0.0               # via apache-beam
-monotonic==1.5            # via fasteners
-more-itertools==8.5.0     # via pytest
-numba==0.51.2             # via librosa, resampy
-numpy==1.19.2             # via apache-beam, librosa, numba, pyarrow, resampy, scikit-learn, scipy
-oauth2client==3.0.0       # via apache-beam, google-apitools
-packaging==20.4           # via pooch, pytest
-pbr==5.5.0                # via mock
+mypy-extensions==0.4.3    # via typing-inspect
+numba==0.52.0             # via librosa, resampy
+numpy==1.19.5             # via apache-beam, librosa, numba, pyarrow, resampy, scikit-learn, scipy
+oauth2client==4.1.3       # via apache-beam, google-apitools
+packaging==20.8           # via pooch, pytest
+pbr==5.5.1                # via mock
 pluggy==0.13.1            # via pytest
-pooch==1.2.0              # via librosa
-protobuf==3.13.0          # via apache-beam, google-api-core, googleapis-common-protos
-py==1.9.0                 # via pytest
-pyarrow==0.17.1           # via apache-beam
+pooch==1.3.0              # via librosa
+proto-plus==1.13.0        # via google-cloud-build
+protobuf==3.14.0          # via apache-beam, google-api-core, googleapis-common-protos, proto-plus
+py==1.10.0                # via pytest
+pyarrow==2.0.0            # via apache-beam
 pyasn1-modules==0.2.8     # via google-auth, oauth2client
 pyasn1==0.4.8             # via oauth2client, pyasn1-modules, rsa
 pycparser==2.20           # via cffi
 pydot==1.4.1              # via apache-beam
-pymongo==3.11.0           # via apache-beam
+pymongo==3.11.2           # via apache-beam
 pyparsing==2.4.7          # via packaging, pydot
-pytest==6.0.2             # via -r job-requirements.in
+pytest==6.2.1             # via -r job-requirements.in
 python-dateutil==2.8.1    # via apache-beam
-pytz==2020.1              # via apache-beam, fastavro, google-api-core
-requests==2.24.0          # via apache-beam, google-api-core, hdfs, pooch
+pytz==2020.5              # via apache-beam, google-api-core
+pyyaml==5.3.1             # via libcst
+requests==2.25.1          # via apache-beam, google-api-core, hdfs, pooch
 resampy==0.2.2            # via librosa
-rsa==4.6                  # via google-auth, oauth2client
-scikit-learn==0.23.2      # via librosa
-scipy==1.5.2              # via librosa, resampy, scikit-learn
-six==1.15.0               # via fasteners, google-api-core, google-apitools, google-auth, google-cloud-bigquery, google-resumable-media, grpcio, hdfs, mock, oauth2client, packaging, protobuf, python-dateutil, resampy
+rsa==4.7                  # via google-auth, oauth2client
+scikit-learn==0.24.0      # via librosa
+scipy==1.6.0              # via librosa, resampy, scikit-learn
+six==1.15.0               # via fasteners, google-api-core, google-apitools, google-auth, google-cloud-bigquery, google-cloud-core, google-resumable-media, grpcio, hdfs, mock, oauth2client, protobuf, python-dateutil, resampy
 soundfile==0.10.3.post1   # via librosa
 threadpoolctl==2.1.0      # via scikit-learn
-toml==0.10.1              # via pytest
-typing-extensions==3.7.4.3  # via apache-beam
-urllib3==1.25.10          # via requests
-zipp==3.1.0               # via importlib-metadata
+toml==0.10.2              # via pytest
+typing-extensions==3.7.4.3  # via apache-beam, libcst, typing-inspect
+typing-inspect==0.6.0     # via libcst
+urllib3==1.26.2           # via requests
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/integration/batch-modular-default/Dockerfile b/integration/batch-modular-default/Dockerfile
@@ -1,4 +1,4 @@
-FROM dataflow.gcr.io/v1beta3/python36-fnapi:2.24.0
+FROM dataflow.gcr.io/v1beta3/python36-fnapi:2.27.0
 
 WORKDIR /usr/src/app
 RUN mkdir -p /usr/src/config
@@ -15,7 +15,7 @@ RUN pip install ./lib
 RUN pip install ./exec
 
 COPY job-requirements.txt job-requirements.txt
-RUN pip install -r job-requirements.txt --use-feature=2020-resolver
+RUN pip install -r job-requirements.txt
 
 COPY __init__.py \
      run.py \
@@ -27,4 +27,4 @@ COPY __init__.py \
 
 ARG KLIO_CONFIG=klio-job.yaml
 COPY $KLIO_CONFIG /usr/src/config/.effective-klio-job.yaml
-RUN pip install . --use-feature=2020-resolver
+RUN pip install .
diff --git a/integration/batch-modular-default/job-requirements.in b/integration/batch-modular-default/job-requirements.in
@@ -1,3 +1,3 @@
-apache-beam[gcp]==2.24.0
+apache-beam[gcp]==2.27.0
 pytest
 # Note: klio packages are installed via docker
diff --git a/integration/batch-modular-default/job-requirements.txt b/integration/batch-modular-default/job-requirements.txt
@@ -4,70 +4,76 @@
 #
 #    pip-compile job-requirements.in
 #
-apache-beam[gcp]==2.24.0  # via -r job-requirements.in
-attrs==20.2.0             # via pytest
+apache-beam[gcp]==2.27.0  # via -r job-requirements.in
+attrs==20.3.0             # via pytest
 avro-python3==1.9.2.1     # via apache-beam
-cachetools==3.1.1         # via apache-beam, google-auth
-certifi==2020.6.20        # via requests
-cffi==1.14.2              # via google-crc32c
-chardet==3.0.4            # via requests
+cachetools==4.2.0         # via apache-beam, google-auth
+certifi==2020.12.5        # via requests
+cffi==1.14.4              # via google-crc32c
+chardet==4.0.0            # via requests
 crcmod==1.7               # via apache-beam
+dataclasses==0.8          # via libcst
 dill==0.3.1.1             # via apache-beam
 docopt==0.6.2             # via hdfs
-fastavro==0.23.6          # via apache-beam
-fasteners==0.15           # via google-apitools
+fastavro==1.2.3           # via apache-beam
+fasteners==0.16           # via google-apitools
 future==0.18.2            # via apache-beam
-google-api-core[grpc,grpcgcp]==1.22.2  # via google-cloud-bigquery, google-cloud-bigtable, google-cloud-core, google-cloud-datastore, google-cloud-dlp, google-cloud-language, google-cloud-pubsub, google-cloud-spanner, google-cloud-videointelligence, google-cloud-vision
+google-api-core[grpc,grpcgcp]==1.24.1  # via google-cloud-bigquery, google-cloud-bigtable, google-cloud-build, google-cloud-core, google-cloud-datastore, google-cloud-dlp, google-cloud-language, google-cloud-pubsub, google-cloud-spanner, google-cloud-videointelligence, google-cloud-vision
 google-apitools==0.5.31   # via apache-beam
-google-auth==1.21.2       # via apache-beam, google-api-core
-google-cloud-bigquery==1.27.2  # via apache-beam
-google-cloud-bigtable==1.4.0  # via apache-beam
-google-cloud-core==1.4.1  # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore, google-cloud-spanner
-google-cloud-datastore==1.15.0  # via apache-beam
+google-auth==1.24.0       # via apache-beam, google-api-core
+google-cloud-bigquery==1.28.0  # via apache-beam
+google-cloud-bigtable==1.6.1  # via apache-beam
+google-cloud-build==2.0.0  # via apache-beam
+google-cloud-core==1.5.0  # via apache-beam, google-cloud-bigquery, google-cloud-bigtable, google-cloud-datastore, google-cloud-spanner
+google-cloud-datastore==1.15.3  # via apache-beam
 google-cloud-dlp==1.0.0   # via apache-beam
 google-cloud-language==1.3.0  # via apache-beam
 google-cloud-pubsub==1.7.0  # via apache-beam
-google-cloud-spanner==1.19.0  # via apache-beam
-google-cloud-videointelligence==1.15.0  # via apache-beam
+google-cloud-spanner==1.19.1  # via apache-beam
+google-cloud-videointelligence==1.16.1  # via apache-beam
 google-cloud-vision==1.0.0  # via apache-beam
-google-crc32c==1.0.0      # via google-resumable-media
-google-resumable-media==1.0.0  # via google-cloud-bigquery
+google-crc32c==1.1.0      # via google-resumable-media
+google-resumable-media==1.2.0  # via google-cloud-bigquery
 googleapis-common-protos[grpc]==1.52.0  # via google-api-core, grpc-google-iam-v1
 grpc-google-iam-v1==0.12.3  # via google-cloud-bigtable, google-cloud-pubsub, google-cloud-spanner
 grpcio-gcp==0.2.2         # via apache-beam, google-api-core
-grpcio==1.32.0            # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, grpcio-gcp
+grpcio==1.34.0            # via apache-beam, google-api-core, googleapis-common-protos, grpc-google-iam-v1, grpcio-gcp
 hdfs==2.5.8               # via apache-beam
 httplib2==0.17.4          # via apache-beam, google-apitools, oauth2client
 idna==2.10                # via requests
-importlib-metadata==1.7.0  # via pluggy, pytest
-iniconfig==1.0.1          # via pytest
+importlib-metadata==3.4.0  # via pluggy, pytest
+iniconfig==1.1.1          # via pytest
+libcst==0.3.16            # via google-cloud-build
 mock==2.0.0               # via apache-beam
-monotonic==1.5            # via fasteners
-more-itertools==8.5.0     # via pytest
-numpy==1.19.2             # via apache-beam, pyarrow
-oauth2client==3.0.0       # via apache-beam, google-apitools
-packaging==20.4           # via pytest
-pbr==5.5.0                # via mock
+mypy-extensions==0.4.3    # via typing-inspect
+numpy==1.19.5             # via apache-beam, pyarrow
+oauth2client==4.1.3       # via apache-beam, google-apitools
+packaging==20.8           # via pytest
+pbr==5.5.1                # via mock
 pluggy==0.13.1            # via pytest
-protobuf==3.13.0          # via apache-beam, google-api-core, googleapis-common-protos
-py==1.9.0                 # via pytest
-pyarrow==0.17.1           # via apache-beam
+proto-plus==1.13.0        # via google-cloud-build
+protobuf==3.14.0          # via apache-beam, google-api-core, googleapis-common-protos, proto-plus
+py==1.10.0                # via pytest
+pyarrow==2.0.0            # via apache-beam
 pyasn1-modules==0.2.8     # via google-auth, oauth2client
 pyasn1==0.4.8             # via oauth2client, pyasn1-modules, rsa
 pycparser==2.20           # via cffi
 pydot==1.4.1              # via apache-beam
-pymongo==3.11.0           # via apache-beam
+pymongo==3.11.2           # via apache-beam
 pyparsing==2.4.7          # via packaging, pydot
-pytest==6.0.2             # via -r job-requirements.in
+pytest==6.2.1             # via -r job-requirements.in
 python-dateutil==2.8.1    # via apache-beam
-pytz==2020.1              # via apache-beam, fastavro, google-api-core
-requests==2.24.0          # via apache-beam, google-api-core, hdfs
-rsa==4.6                  # via google-auth, oauth2client
-six==1.15.0               # via fasteners, google-api-core, google-apitools, google-auth, google-cloud-bigquery, google-resumable-media, grpcio, hdfs, mock, oauth2client, packaging, protobuf, python-dateutil
-toml==0.10.1              # via pytest
-typing-extensions==3.7.4.3  # via apache-beam
-urllib3==1.25.10          # via requests
-zipp==3.1.0               # via importlib-metadata
+pytz==2020.5              # via apache-beam, google-api-core
+pyyaml==5.3.1             # via libcst
+requests==2.25.1          # via apache-beam, google-api-core, hdfs
+rsa==4.7                  # via google-auth, oauth2client
+six==1.15.0               # via fasteners, google-api-core, google-apitools, google-auth, google-cloud-bigquery, google-cloud-core, google-resumable-media, grpcio, hdfs, mock, oauth2client, protobuf, python-dateutil
+toml==0.10.2              # via pytest
+typing-extensions==3.7.4.3  # via apache-beam, importlib-metadata, libcst, typing-inspect
+typing-inspect==0.6.0     # via libcst
+urllib3==1.26.2           # via requests
+zipp==3.4.0               # via importlib-metadata
+
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/integration/multi-event-input-batch/Dockerfile b/integration/multi-event-input-batch/Dockerfile
@@ -1,4 +1,4 @@
-FROM dataflow.gcr.io/v1beta3/python36-fnapi:2.24.0
+FROM dataflow.gcr.io/v1beta3/python36-fnapi:2.27.0
 
 WORKDIR /usr/src/app
 RUN mkdir -p /usr/src/config
@@ -15,7 +15,7 @@ RUN pip install ./lib
 RUN pip install ./exec
 
 COPY job-requirements.txt job-requirements.txt
-RUN pip install -r job-requirements.txt --use-feature=2020-resolver
+RUN pip install -r job-requirements.txt
 
 COPY __init__.py \
      run.py \
@@ -27,4 +27,4 @@ COPY __init__.py \
 
 ARG KLIO_CONFIG=klio-job.yaml
 COPY $KLIO_CONFIG /usr/src/config/.effective-klio-job.yaml
-RUN pip install . --use-feature=2020-resolver
+RUN pip install .
diff --git a/integration/multi-event-input-batch/job-requirements.in b/integration/multi-event-input-batch/job-requirements.in
@@ -1,3 +1,3 @@
-apache-beam[gcp]==2.24.0
+apache-beam[gcp]==2.27.0
 pytest
 # Note: klio packages are installed via docker