Enable Python code coverage for onnx runs (pytorch#47387)

Summary:
Fixes pytorch#44120

Pull Request resolved: pytorch#47387

Reviewed By: heitorschueroff

Differential Revision: D24737378

Pulled By: janeyx99

fbshipit-source-id: 79e3d0b62f7da0617330f312fb1ed548c6be2a3b

.circleci/config.yml

@@ -655,11 +655,13 @@ jobs:
           echo "Retrieving test reports"
           docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!'
           if [[ ${BUILD_ENVIRONMENT} == *"coverage"* ]]; then
+              echo "Retrieving C++ coverage report"
+              docker cp $id:/var/lib/jenkins/workspace/build/coverage.info ./test
+          fi
+          if [[ ${BUILD_ENVIRONMENT} == *"coverage"* || ${BUILD_ENVIRONMENT} == *"onnx"* ]]; then
               echo "Retrieving Python coverage report"
               docker cp $id:/var/lib/jenkins/workspace/test/.coverage ./test
               docker cp $id:/var/lib/jenkins/workspace/test/coverage.xml ./test
-              echo "Retrieving C++ coverage report"
-              docker cp $id:/var/lib/jenkins/workspace/build/coverage.info ./test
               python3 -mpip install codecov
               python3 -mcodecov
           fi

.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml

@@ -217,11 +217,13 @@ jobs:
           echo "Retrieving test reports"
           docker cp $id:/var/lib/jenkins/workspace/test/test-reports ./ || echo 'No test reports found!'
           if [[ ${BUILD_ENVIRONMENT} == *"coverage"* ]]; then
+              echo "Retrieving C++ coverage report"
+              docker cp $id:/var/lib/jenkins/workspace/build/coverage.info ./test
+          fi
+          if [[ ${BUILD_ENVIRONMENT} == *"coverage"* || ${BUILD_ENVIRONMENT} == *"onnx"* ]]; then
               echo "Retrieving Python coverage report"
               docker cp $id:/var/lib/jenkins/workspace/test/.coverage ./test
               docker cp $id:/var/lib/jenkins/workspace/test/coverage.xml ./test
-              echo "Retrieving C++ coverage report"
-              docker cp $id:/var/lib/jenkins/workspace/build/coverage.info ./test
               python3 -mpip install codecov
               python3 -mcodecov
           fi

.jenkins/caffe2/test.sh

@@ -93,6 +93,7 @@ fi
   # See comments on
   # https://github.com/HypothesisWorks/hypothesis-python/commit/eadd62e467d6cee6216e71b391951ec25b4f5830
   $MAYBE_SUDO pip -q uninstall -y hypothesis
+  $MAYBE_SUDO pip -q uninstall -y coverage
   # "pip install hypothesis==3.44.6" from official server is unreliable on
   # CircleCI, so we host a copy on S3 instead
   $MAYBE_SUDO pip -q install attrs==18.1.0 -f https://s3.amazonaws.com/ossci-linux/wheels/attrs-18.1.0-py2.py3-none-any.whl

scripts/onnx/test.sh

@@ -23,12 +23,13 @@ do
 done
 set -- "${UNKNOWN[@]}" # leave UNKNOWN
 
-pip install pytest scipy hypothesis # these are all already satisfied in CI
-
 if [[ $PARALLEL == 1 ]]; then
     pip install pytest-xdist
 fi
 
+pip install pytest scipy hypothesis # these may not be necessary
+pip install pytest-cov # installing since `coverage run -m pytest ..` doesn't work
+
 # realpath might not be available on MacOS
 script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
 top_dir=$(dirname $(dirname $(dirname "$script_path")))
@@ -38,6 +39,10 @@ test_paths=(
 
 args=()
 args+=("-v")
+args+=("--cov")
+args+=("--cov-report")
+args+=("xml:test/coverage.xml")
+args+=("--cov-append")
 if [[ $PARALLEL == 1 ]]; then
   args+=("-n")
   args+=("3")
@@ -74,3 +79,6 @@ if [[ "$BUILD_ENVIRONMENT" == *ort_test2* ]]; then
   pytest "${args[@]}" \
     "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py::TestONNXRuntime_opset12_onnx_shape_inference"
 fi
+
+# Our CI expects both coverage.xml and .coverage to be within test/
+mv .coverage test/.coverage