Airspeed Velocity (asv) performance benchmarks setup: part-1 (quantumlib#3822)

* adds asv benchmarks, asv config, readme.md
* removes old pytest-benchmarks, configs and requirements.

Author: tanujkhattar

.dockerignore

@@ -29,8 +29,8 @@ rtd_docs/generated
 # Mac only
 .DS_Store
 
-# From pytest-benchmark
-*.benchmarks/
+# From asv
+*.asv/
 
 # Ignore generated Jupyter files
 *.ipynb_checkpoints/

.github/workflows/ci.yml

@@ -121,7 +121,7 @@ jobs:
           # TODO: move to requirements.txt after #3704
           pip install codeowners==0.1.2 # linux only package
       - name: Pytest check
-        run: check/pytest --ignore=cirq/contrib --benchmark-skip --actually-quiet
+        run: check/pytest --ignore=cirq/contrib --actually-quiet
   build_docs:
     name: Build docs
     runs-on: ubuntu-16.04
@@ -189,7 +189,7 @@ jobs:
           pip install -r requirements.txt
           pip install -r dev_tools/conf/pip-list-dev-tools.txt
       - name: Pytest Windows
-        run: check/pytest --ignore=cirq/contrib --benchmark-skip --actually-quiet
+        run: check/pytest --ignore=cirq/contrib --actually-quiet
         shell: bash
   macos:
     name: Pytest MacOS
@@ -208,7 +208,7 @@ jobs:
           pip install -r requirements.txt
           pip install -r dev_tools/conf/pip-list-dev-tools.txt
       - name: Pytest check
-        run: check/pytest --ignore=cirq/contrib --benchmark-skip
+        run: check/pytest --ignore=cirq/contrib
   notebooks-stable:
     name: Changed Notebooks Isolated Test against Cirq stable
     runs-on: ubuntu-16.04

.gitignore

@@ -32,8 +32,8 @@ docs/api_docs
 # Mac only
 .DS_Store
 
-# From pytest-benchmark
-*.benchmarks/
+# From asv
+*.asv/
 
 # Ignore generated Jupyter files
 *.ipynb_checkpoints/

asv.conf.json

@@ -0,0 +1,17 @@
+{
+    // See https://asv.readthedocs.io/en/stable/asv.conf.json.html for documentation.
+    "version": 1,
+    "project": "Cirq",
+    "project_url": "https://quantumai.google/cirq",
+    "repo": ".",
+    "branches": ["master"],
+    "dvcs": "git",
+    "environment_type": "virtualenv",
+    "show_commit_url": "https://github.com/quantumlib/Cirq/commit/",
+    "pythons": ["3.6", "3.7", "3.8"],
+    "benchmark_dir": "benchmarks",
+    "env_dir": ".asv/env",
+    "results_dir": ".asv/results",
+    "html_dir": ".asv/html",
+    "hash_length": 8,
+}

benchmarks/README.md

@@ -0,0 +1,22 @@
+# Cirq Performance Benchmarks
+Performance benchmarking Cirq with [Airspeed Velocity](https://asv.readthedocs.io/en/stable/index.html).
+
+## Overview
+The benchmark files (`bench_*.py`) stored in the current package (`benchmarks/*`) are used by asv to run benchmark tests for Cirq. For more information on how to write new benchmarks, please refer [Writing benchmarks guide by ASV](https://asv.readthedocs.io/en/stable/writing_benchmarks.html)
+
+## Usage
+To run all benchmarks, navigate to the root Cirq directory at the command line and execute
+
+```bash
+./check/asv_run
+```
+
+You can also pass arguments to the script, which would be forwarded to the `asv run` command. For eg:
+```bash
+./check/asv_run --quick --bench bench_examples --python 3.8
+```
+
+Please refer [Running Benchmarks guide by ASV](https://asv.readthedocs.io/en/stable/using.html#running-benchmarks) for more information. 
+
+## Results Database
+TODO([#3838](https://github.com/quantumlib/Cirq/issues/3838)): Add details regarding GCP setup.

benchmarks/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2021 The Cirq Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

benchmarks/bench_examples.py

@@ -0,0 +1,63 @@
+# Copyright 2021 The Cirq Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sys import path
+from os.path import dirname as dir
+
+path.append(dir(path[0]))
+
+import examples.bernstein_vazirani
+import examples.place_on_bristlecone
+import examples.hello_qubit
+import examples.bell_inequality
+import examples.quantum_fourier_transform
+import examples.bcs_mean_field
+import examples.grover
+import examples.phase_estimator
+import examples.quantum_teleportation
+import examples.superdense_coding
+
+
+class ExamplesTest:
+    """Benchmark algorithms in Cirq/examples/ using ASV."""
+
+    def time_example_runs_bernstein_vazirani_perf(self):
+        examples.bernstein_vazirani.main(qubit_count=3)
+
+    def time_example_runs_hello_line_perf(self):
+        examples.place_on_bristlecone.main()
+
+    def time_example_runs_hello_qubit_perf(self):
+        examples.hello_qubit.main()
+
+    def time_example_runs_bell_inequality_perf(self):
+        examples.bell_inequality.main()
+
+    def time_example_runs_quantum_fourier_transform_perf(self):
+        examples.quantum_fourier_transform.main()
+
+    def time_example_runs_bcs_mean_field_perf(self):
+        examples.bcs_mean_field.main()
+
+    def time_example_runs_grover_perf(self):
+        examples.grover.main()
+
+    def time_example_runs_phase_estimator_perf(self):
+        examples.phase_estimator.main(qnums=(2,), repetitions=2)
+
+    def time_example_runs_quantum_teleportation(self):
+        examples.quantum_teleportation.main()
+
+    def time_example_runs_superdense_coding(self):
+        examples.superdense_coding.main()

benchmarks/bench_linalg_decompositions.py

@@ -0,0 +1,44 @@
+# Copyright 2021 The Cirq Developers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+import cirq
+
+# yapf: disable
+SWAP = np.array([[1, 0, 0, 0],
+                [0, 0, 1, 0],
+                [0, 1, 0, 0],
+                [0, 0, 0, 1]])
+CNOT = np.array([[1, 0, 0, 0],
+                [0, 1, 0, 0],
+                [0, 0, 0, 1],
+                [0, 0, 1, 0]])
+CZ = np.diag([1, 1, 1, -1])
+# yapf: enable
+
+
+def time_kak_decomposition(target):
+    """Benchmark kak_decomposition
+    kak_decomposition is benchmarked because it was historically slow.
+    See https://github.com/quantumlib/Cirq/issues/3840 for status of other benchmarks.
+    """
+    cirq.kak_decomposition(target)
+
+
+time_kak_decomposition.params = [  # type: ignore
+    [np.eye(4), SWAP, SWAP * 1j, CZ, CNOT, SWAP.dot(CZ)]
+    + [cirq.testing.random_unitary(4) for _ in range(10)]
+]
+time_kak_decomposition.param_names = ["gate"]  # type: ignore

check/performance check/asv_run

@@ -1,16 +1,15 @@
 #!/usr/bin/env bash
 
 ################################################################################
-# Runs performance tests
+# Runs performance benchmark tests using asv
 #
 # Usage:
-#     check/performance
+#     check/asv_run [--flags for asv run]
 ################################################################################
 
 # Get the working directory to the repo root.
 cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd "$(git rev-parse --show-toplevel)"
 
-# Test will fail if Mean is 0.1 seconds slower for any test.
-pytest --benchmark-autosave --benchmark-only --benchmark-compare --benchmark-compare-fail=mean:0.1  \
-&& cd .benchmarks/Linux-CPython-3.6-64bit/ && ls -t | tail -n +2 | xargs rm -- && ls
+export ASV_PYTHONPATH="$(git rev-parse --show-toplevel)/examples"
+asv run $@

check/pytest-and-incremental-coverage

@@ -56,8 +56,7 @@ check/pytest . \
     --actually-quiet \
     --cov \
     --cov-report=annotate \
-    --cov-config=dev_tools/conf/.coveragerc \
-    --benchmark-skip
+    --cov-config=dev_tools/conf/.coveragerc
 pytest_result=$?
 
 # Analyze coverage files.

check/pytest-changed-files-and-incremental-coverage

@@ -87,8 +87,7 @@ fi
 check/pytest "${changed_python_tests[@]}" \
     "${cov_changed_python_file_dirs[@]}" \
     --cov-report=annotate \
-    --cov-config=dev_tools/conf/.coveragerc \
-    --benchmark-skip
+    --cov-config=dev_tools/conf/.coveragerc
 pytest_result=$?
 
 # Analyze coverage files.

cirq/linalg/decompositions_perf_test.py

@@ -352,7 +352,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.out == (
         'INTERCEPTED check/pytest '
         '. --actually-quiet --cov --cov-report=annotate '
-        '--cov-config=dev_tools/conf/.coveragerc --benchmark-skip\n'
+        '--cov-config=dev_tools/conf/.coveragerc\n'
         'INTERCEPTED '
         'python dev_tools/check_incremental_coverage_annotations.py HEAD\n'
     )
@@ -376,7 +376,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.out == (
         'INTERCEPTED check/pytest '
         '. --actually-quiet --cov --cov-report=annotate '
-        '--cov-config=dev_tools/conf/.coveragerc --benchmark-skip\n'
+        '--cov-config=dev_tools/conf/.coveragerc\n'
         'INTERCEPTED python '
         'dev_tools/check_incremental_coverage_annotations.py master\n'
     )
@@ -392,7 +392,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.out == (
         'INTERCEPTED check/pytest '
         '. --actually-quiet --cov --cov-report=annotate '
-        '--cov-config=dev_tools/conf/.coveragerc --benchmark-skip\n'
+        '--cov-config=dev_tools/conf/.coveragerc\n'
         'INTERCEPTED python '
         'dev_tools/check_incremental_coverage_annotations.py origin/master\n'
     )
@@ -408,7 +408,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.out == (
         'INTERCEPTED check/pytest '
         '. --actually-quiet --cov --cov-report=annotate '
-        '--cov-config=dev_tools/conf/.coveragerc --benchmark-skip\n'
+        '--cov-config=dev_tools/conf/.coveragerc\n'
         'INTERCEPTED python '
         'dev_tools/check_incremental_coverage_annotations.py upstream/master\n'
     )
@@ -424,7 +424,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.out == (
         'INTERCEPTED check/pytest '
         '. --actually-quiet --cov --cov-report=annotate '
-        '--cov-config=dev_tools/conf/.coveragerc --benchmark-skip\n'
+        '--cov-config=dev_tools/conf/.coveragerc\n'
         'INTERCEPTED python '
         'dev_tools/check_incremental_coverage_annotations.py upstream/master\n'
     )
@@ -452,7 +452,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.out == (
         'INTERCEPTED check/pytest '
         '. --actually-quiet --cov --cov-report=annotate '
-        '--cov-config=dev_tools/conf/.coveragerc --benchmark-skip\n'
+        '--cov-config=dev_tools/conf/.coveragerc\n'
         'INTERCEPTED python '
         'dev_tools/check_incremental_coverage_annotations.py HEAD\n'
     )
@@ -468,7 +468,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.out == (
         'INTERCEPTED check/pytest '
         '. --actually-quiet --cov --cov-report=annotate '
-        '--cov-config=dev_tools/conf/.coveragerc --benchmark-skip\n'
+        '--cov-config=dev_tools/conf/.coveragerc\n'
         'INTERCEPTED python '
         'dev_tools/check_incremental_coverage_annotations.py master\n'
     )
@@ -491,7 +491,7 @@ def test_pytest_and_incremental_coverage_branch_selection(tmpdir_factory):
     assert result.out.startswith(
         'INTERCEPTED check/pytest '
         '. --actually-quiet --cov --cov-report=annotate '
-        '--cov-config=dev_tools/conf/.coveragerc --benchmark-skip\n'
+        '--cov-config=dev_tools/conf/.coveragerc\n'
         'INTERCEPTED python '
         'dev_tools/check_incremental_coverage_annotations.py '
     )

dev_tools/bash_scripts_test.py

@@ -3,3 +3,6 @@
 # Note: this means coverage must be run from the cirq repo root.
 # Failure to do so will result in false positives.
 include = ./*
+
+omit = 
+    benchmarks/*

dev_tools/conf/.coveragerc

@@ -1,11 +1,13 @@
+
 # For testing and analyzing code.
+asv
+virtualenv
 black==20.8b1
 mypy~=0.782.0
 pylint~=2.6.0
 pytest~=5.4.1
 pytest-asyncio~=0.12.0
 pytest-cov~=2.5.0
-pytest-benchmark~=3.2.0
 filelock~=3.0.12
 
 # For generating protobufs

dev_tools/conf/pip-list-dev-tools.txt

@@ -26,6 +26,7 @@
     r'^setup\.py$',  # Installation code.
     r'^cirq/google/engine/client/.+.py$',  # Generated gRPC client code.
     r'^cirq/google/api/v1/.+.py$',  # deprecated API code
+    r'^benchmarks/',
 ]
 IGNORED_BLOCK_PATTERNS = [
     r'^\s*if TYPE_CHECKING:$',  # imports needed only while type-checking.

dev_tools/incremental_coverage.py

@@ -94,7 +94,7 @@ for PYTHON_VERSION in python3; do
 
     # Run tests.
     echo Installing pytest requirements
-    "${tmp_dir}/${PYTHON_VERSION}/bin/pip" install --quiet pytest pytest-benchmark pytest-asyncio
+    "${tmp_dir}/${PYTHON_VERSION}/bin/pip" install --quiet pytest pytest-asyncio
     PY_VER=$(ls "${tmp_dir}/${PYTHON_VERSION}/lib")
     echo Running cirq tests
     cirq_dir="${tmp_dir}/${PYTHON_VERSION}/lib/${PY_VER}/site-packages/${PROJECT_NAME}"