Add performance benchmarking scripts (projectmesa#1979)

* benchmarks: Upload initial models from JuliaDynamics

https://github.com/JuliaDynamics/ABMFrameworksComparison/tree/5551d7abf1611d377b3b32346c7774f176af4c65

* benchmarks: Add dictionary with configurations

* benchmarks: Update configurations, use relative imports

* benchmarks: Add single script to run all benchmarks

* benchmarks: Update configurations

Few less replications, some more seeds. Every benchmark now takes between 10 and 20 seconds (on my machine).

* benchmarks: Add generated pickle files to gitignore

That allows switching branches without benchmarks results disappearing

* benchmarks: Update global script to calculate and save stuff

Prints some stuff when running and saves a pickle file after running.

* benchmarks: Write a script to statistically compare runs

- The bootstrap_speedup_confidence_interval function calculates the mean speedup and its confidence interval using bootstrapping, which is more suitable for paired data.
- The mean speedup and confidence interval are calculated for both initialization and run times.
- Positive values indicate an increase in time (longer duration), and negative values indicate a decrease (shorter duration).
- The results are displayed in a DataFrame with the percentage changes and their confidence intervals.

* benchmarks: Remove seperate benchmark scripts

* benchmarks: Black and ruff

* benchmarks: Use old f-string formatting to work with Python 3.11 and older

* benchmarks: Add fancy colored performance indicators 🟢🔴🔵

If the 95% confidence interval is:
- fully below -3%: 🟢
- fully above +3%: 🔴
- else: 🔵

* Fix typos caught by codespell

* Apply ruff format benchmarks/

* Apply ruff --fix benchmarks/

* Apply ruff --fix --unsafe-fixes benchmarks/

* Apply manual Ruff fixes

* codecov: Ignore benchmarks/

---------

Co-authored-by: rht <[email protected]>

Author: EwoutH

.gitignore

@@ -1,3 +1,6 @@
+# Benchmarking
+benchmarking/**/*.pickle
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

benchmarks/Flocking/Flocking.py

@@ -0,0 +1,80 @@
+"""
+Flockers
+=============================================================
+A Mesa implementation of Craig Reynolds's Boids flocker model.
+Uses numpy arrays to represent vectors.
+"""
+
+import numpy as np
+
+from mesa import Model
+from mesa.space import ContinuousSpace
+from mesa.time import RandomActivation
+
+from .boid import Boid
+
+
+class BoidFlockers(Model):
+    """
+    Flocker model class. Handles agent creation, placement and scheduling.
+    """
+
+    def __init__(
+        self,
+        seed,
+        population,
+        width,
+        height,
+        vision,
+        speed=1,
+        separation=1,
+        cohere=0.03,
+        separate=0.015,
+        match=0.05,
+    ):
+        """
+        Create a new Flockers model.
+
+        Args:
+            population: Number of Boids
+            width, height: Size of the space.
+            speed: How fast should the Boids move.
+            vision: How far around should each Boid look for its neighbors
+            separation: What's the minimum distance each Boid will attempt to
+                    keep from any other
+            cohere, separate, match: factors for the relative importance of
+                    the three drives."""
+        super().__init__(seed=seed)
+        self.population = population
+        self.vision = vision
+        self.speed = speed
+        self.separation = separation
+        self.schedule = RandomActivation(self)
+        self.space = ContinuousSpace(width, height, True)
+        self.factors = {"cohere": cohere, "separate": separate, "match": match}
+        self.make_agents()
+
+    def make_agents(self):
+        """
+        Create self.population agents, with random positions and starting headings.
+        """
+        for i in range(self.population):
+            x = self.random.random() * self.space.x_max
+            y = self.random.random() * self.space.y_max
+            pos = np.array((x, y))
+            velocity = np.random.random(2) * 2 - 1
+            boid = Boid(
+                i,
+                self,
+                pos,
+                self.speed,
+                velocity,
+                self.vision,
+                self.separation,
+                **self.factors,
+            )
+            self.space.place_agent(boid, pos)
+            self.schedule.add(boid)
+
+    def step(self):
+        self.schedule.step()

benchmarks/Flocking/__init__.py

@@ -0,0 +1,81 @@
+import numpy as np
+
+from mesa import Agent
+
+
+class Boid(Agent):
+    """
+    A Boid-style flocker agent.
+
+    The agent follows three behaviors to flock:
+        - Cohesion: steering towards neighboring agents.
+        - Separation: avoiding getting too close to any other agent.
+        - Alignment: try to fly in the same direction as the neighbors.
+
+    Boids have a vision that defines the radius in which they look for their
+    neighbors to flock with. Their speed (a scalar) and velocity (a vector)
+    define their movement. Separation is their desired minimum distance from
+    any other Boid.
+    """
+
+    def __init__(
+        self,
+        unique_id,
+        model,
+        pos,
+        speed,
+        velocity,
+        vision,
+        separation,
+        cohere=0.03,
+        separate=0.015,
+        match=0.05,
+    ):
+        """
+        Create a new Boid flocker agent.
+
+        Args:
+            unique_id: Unique agent identifier.
+            pos: Starting position
+            speed: Distance to move per step.
+            heading: numpy vector for the Boid's direction of movement.
+            vision: Radius to look around for nearby Boids.
+            separation: Minimum distance to maintain from other Boids.
+            cohere: the relative importance of matching neighbors' positions
+            separate: the relative importance of avoiding close neighbors
+            match: the relative importance of matching neighbors' headings
+
+        """
+        super().__init__(unique_id, model)
+        self.pos = np.array(pos)
+        self.speed = speed
+        self.velocity = velocity
+        self.vision = vision
+        self.separation = separation
+        self.cohere_factor = cohere
+        self.separate_factor = separate
+        self.match_factor = match
+
+    def step(self):
+        """
+        Get the Boid's neighbors, compute the new vector, and move accordingly.
+        """
+
+        neighbors = self.model.space.get_neighbors(self.pos, self.vision, False)
+        n = 0
+        match_vector, separation_vector, cohere = np.zeros((3, 2))
+        for neighbor in neighbors:
+            n += 1
+            heading = self.model.space.get_heading(self.pos, neighbor.pos)
+            cohere += heading
+            if self.model.space.get_distance(self.pos, neighbor.pos) < self.separation:
+                separation_vector -= heading
+            match_vector += neighbor.velocity
+        n = max(n, 1)
+        cohere = cohere * self.cohere_factor
+        separation_vector = separation_vector * self.separate_factor
+        match_vector = match_vector * self.match_factor
+        self.velocity += (cohere + separation_vector + match_vector) / n
+        self.velocity /= np.linalg.norm(self.velocity)
+        new_pos = self.pos + self.velocity * self.speed
+        self.model.space.move_agent(self, new_pos)

benchmarks/Flocking/boid.py

@@ -0,0 +1,77 @@
+import random
+
+from mesa import Agent, Model
+from mesa.space import SingleGrid
+from mesa.time import RandomActivation
+
+
+class SchellingAgent(Agent):
+    """
+    Schelling segregation agent
+    """
+
+    def __init__(self, pos, model, agent_type):
+        """
+        Create a new Schelling agent.
+        Args:
+           unique_id: Unique identifier for the agent.
+           x, y: Agent initial location.
+           agent_type: Indicator for the agent's type (minority=1, majority=0)
+        """
+        super().__init__(pos, model)
+        self.pos = pos
+        self.type = agent_type
+
+    def step(self):
+        similar = 0
+        r = self.model.radius
+        for neighbor in self.model.grid.iter_neighbors(self.pos, moore=True, radius=r):
+            if neighbor.type == self.type:
+                similar += 1
+
+        # If unhappy, move:
+        if similar < self.model.homophily:
+            self.model.grid.move_to_empty(self)
+        else:
+            self.model.happy += 1
+
+
+class SchellingModel(Model):
+    """
+    Model class for the Schelling segregation model.
+    """
+
+    def __init__(
+        self, seed, height, width, homophily, radius, density, minority_pc=0.5
+    ):
+        """ """
+        super().__init__(seed=seed)
+        self.height = height
+        self.width = width
+        self.density = density
+        self.minority_pc = minority_pc
+        self.homophily = homophily
+        self.radius = radius
+
+        self.schedule = RandomActivation(self)
+        self.grid = SingleGrid(height, width, torus=True)
+
+        self.happy = 0
+
+        # Set up agents
+        # We use a grid iterator that returns
+        # the coordinates of a cell as well as
+        # its contents. (coord_iter)
+        for _cont, pos in self.grid.coord_iter():
+            if random.random() < self.density:  # noqa: S311
+                agent_type = 1 if random.random() < self.minority_pc else 0  # noqa: S311
+                agent = SchellingAgent(pos, self, agent_type)
+                self.grid.place_agent(agent, pos)
+                self.schedule.add(agent)
+
+    def step(self):
+        """
+        Run one step of the model.
+        """
+        self.happy = 0  # Reset counter of happy agents
+        self.schedule.step()

benchmarks/Schelling/Schelling.py

@@ -0,0 +1,103 @@
+"""
+Wolf-Sheep Predation Model
+================================
+
+Replication of the model found in NetLogo:
+    Wilensky, U. (1997). NetLogo Wolf Sheep Predation model.
+    http://ccl.northwestern.edu/netlogo/models/WolfSheepPredation.
+    Center for Connected Learning and Computer-Based Modeling,
+    Northwestern University, Evanston, IL.
+"""
+
+import mesa
+from mesa.space import MultiGrid
+from mesa.time import RandomActivationByType
+
+from .agents import GrassPatch, Sheep, Wolf
+
+
+class WolfSheep(mesa.Model):
+    """
+    Wolf-Sheep Predation Model
+
+    A model for simulating wolf and sheep (predator-prey) ecosystem modelling.
+    """
+
+    def __init__(
+        self,
+        seed,
+        height,
+        width,
+        initial_sheep,
+        initial_wolves,
+        sheep_reproduce,
+        wolf_reproduce,
+        grass_regrowth_time,
+        wolf_gain_from_food=13,
+        sheep_gain_from_food=5,
+    ):
+        """
+        Create a new Wolf-Sheep model with the given parameters.
+
+        Args:
+            initial_sheep: Number of sheep to start with
+            initial_wolves: Number of wolves to start with
+            sheep_reproduce: Probability of each sheep reproducing each step
+            wolf_reproduce: Probability of each wolf reproducing each step
+            wolf_gain_from_food: Energy a wolf gains from eating a sheep
+            grass: Whether to have the sheep eat grass for energy
+            grass_regrowth_time: How long it takes for a grass patch to regrow
+                                 once it is eaten
+            sheep_gain_from_food: Energy sheep gain from grass, if enabled.
+        """
+        super().__init__(seed=seed)
+        # Set parameters
+        self.height = height
+        self.width = width
+        self.initial_sheep = initial_sheep
+        self.initial_wolves = initial_wolves
+        self.sheep_reproduce = sheep_reproduce
+        self.wolf_reproduce = wolf_reproduce
+        self.wolf_gain_from_food = wolf_gain_from_food
+        self.grass_regrowth_time = grass_regrowth_time
+        self.sheep_gain_from_food = sheep_gain_from_food
+
+        self.schedule = RandomActivationByType(self)
+        self.grid = MultiGrid(self.height, self.width, torus=False)
+
+        # Create sheep:
+        for _i in range(self.initial_sheep):
+            pos = (
+                self.random.randrange(self.width),
+                self.random.randrange(self.height),
+            )
+            energy = self.random.randrange(2 * self.sheep_gain_from_food)
+            sheep = Sheep(self.next_id(), pos, self, True, energy)
+            self.grid.place_agent(sheep, pos)
+            self.schedule.add(sheep)
+
+        # Create wolves
+        for _i in range(self.initial_wolves):
+            pos = (
+                self.random.randrange(self.width),
+                self.random.randrange(self.height),
+            )
+            energy = self.random.randrange(2 * self.wolf_gain_from_food)
+            wolf = Wolf(self.next_id(), pos, self, True, energy)
+            self.grid.place_agent(wolf, pos)
+            self.schedule.add(wolf)
+
+        # Create grass patches
+        possibly_fully_grown = [True, False]
+        for _agent, pos in self.grid.coord_iter():
+            fully_grown = self.random.choice(possibly_fully_grown)
+            if fully_grown:
+                countdown = self.grass_regrowth_time
+            else:
+                countdown = self.random.randrange(self.grass_regrowth_time)
+            patch = GrassPatch(self.next_id(), pos, self, fully_grown, countdown)
+            self.grid.place_agent(patch, pos)
+            self.schedule.add(patch)
+
+    def step(self):
+        self.schedule.step()

benchmarks/Schelling/__init__.py

@@ -0,0 +1,111 @@
+from mesa import Agent
+
+from .random_walk import RandomWalker
+
+
+class Sheep(RandomWalker):
+    """
+    A sheep that walks around, reproduces (asexually) and gets eaten.
+
+    The init is the same as the RandomWalker.
+    """
+
+    def __init__(self, unique_id, pos, model, moore, energy=None):
+        super().__init__(unique_id, pos, model, moore=moore)
+        self.energy = energy
+
+    def step(self):
+        """
+        A model step. Move, then eat grass and reproduce.
+        """
+        self.random_move()
+
+        # Reduce energy
+        self.energy -= 1
+
+        # If there is grass available, eat it
+        this_cell = self.model.grid.get_cell_list_contents(self.pos)
+        grass_patch = next(obj for obj in this_cell if isinstance(obj, GrassPatch))
+        if grass_patch.fully_grown:
+            self.energy += self.model.sheep_gain_from_food
+            grass_patch.fully_grown = False
+
+        # Death
+        if self.energy < 0:
+            self.model.grid.remove_agent(self)
+            self.model.schedule.remove(self)
+        elif self.random.random() < self.model.sheep_reproduce:
+            # Create a new sheep:
+            self.energy /= 2
+            lamb = Sheep(
+                self.model.next_id(), self.pos, self.model, self.moore, self.energy
+            )
+            self.model.grid.place_agent(lamb, self.pos)
+            self.model.schedule.add(lamb)
+
+
+class Wolf(RandomWalker):
+    """
+    A wolf that walks around, reproduces (asexually) and eats sheep.
+    """
+
+    def __init__(self, unique_id, pos, model, moore, energy=None):
+        super().__init__(unique_id, pos, model, moore=moore)
+        self.energy = energy
+
+    def step(self):
+        self.random_move()
+        self.energy -= 1
+
+        # If there are sheep present, eat one
+        x, y = self.pos
+        this_cell = self.model.grid.get_cell_list_contents([self.pos])
+        sheep = [obj for obj in this_cell if isinstance(obj, Sheep)]
+        if len(sheep) > 0:
+            sheep_to_eat = self.random.choice(sheep)
+            self.energy += self.model.wolf_gain_from_food
+
+            # Kill the sheep
+            self.model.grid.remove_agent(sheep_to_eat)
+            self.model.schedule.remove(sheep_to_eat)
+
+        # Death or reproduction
+        if self.energy < 0:
+            self.model.grid.remove_agent(self)
+            self.model.schedule.remove(self)
+        elif self.random.random() < self.model.wolf_reproduce:
+            # Create a new wolf cub
+            self.energy /= 2
+            cub = Wolf(
+                self.model.next_id(), self.pos, self.model, self.moore, self.energy
+            )
+            self.model.grid.place_agent(cub, cub.pos)
+            self.model.schedule.add(cub)
+
+
+class GrassPatch(Agent):
+    """
+    A patch of grass that grows at a fixed rate and it is eaten by sheep
+    """
+
+    def __init__(self, unique_id, pos, model, fully_grown, countdown):
+        """
+        Creates a new patch of grass
+
+        Args:
+            grown: (boolean) Whether the patch of grass is fully grown or not
+            countdown: Time for the patch of grass to be fully grown again
+        """
+        super().__init__(unique_id, model)
+        self.fully_grown = fully_grown
+        self.countdown = countdown
+        self.pos = pos
+
+    def step(self):
+        if not self.fully_grown:
+            if self.countdown <= 0:
+                # Set as fully grown
+                self.fully_grown = True
+                self.countdown = self.model.grass_regrowth_time
+            else:
+                self.countdown -= 1

benchmarks/WolfSheep/WolfSheep.py

@@ -0,0 +1,37 @@
+"""
+Generalized behavior for random walking, one grid cell at a time.
+"""
+
+from mesa import Agent
+
+
+class RandomWalker(Agent):
+    """
+    Class implementing random walker methods in a generalized manner.
+
+    Not intended to be used on its own, but to inherit its methods to multiple
+    other agents.
+
+    """
+
+    def __init__(self, unique_id, pos, model, moore=True):
+        """
+        grid: The MultiGrid object in which the agent lives.
+        x: The agent's current x coordinate
+        y: The agent's current y coordinate
+        moore: If True, may move in all 8 directions.
+                Otherwise, only up, down, left, right.
+        """
+        super().__init__(unique_id, model)
+        self.pos = pos
+        self.moore = moore
+
+    def random_move(self):
+        """
+        Step one cell in any allowable direction.
+        """
+        # Pick the next cell from the adjacent cells.
+        next_moves = self.model.grid.get_neighborhood(self.pos, self.moore, True)
+        next_move = self.random.choice(next_moves)
+        # Now move:
+        self.model.grid.move_agent(self, next_move)

benchmarks/WolfSheep/__init__.py

@@ -0,0 +1,87 @@
+import pickle
+
+import numpy as np
+import pandas as pd
+
+filename1 = "timings_1"
+filename2 = "timings_2"
+
+with open(f"{filename1}.pickle", "rb") as handle:
+    timings_1 = pickle.load(handle)  # noqa: S301
+
+with open(f"{filename2}.pickle", "rb") as handle:
+    timings_2 = pickle.load(handle)  # noqa: S301
+
+
+# Function to calculate the percentage change and perform bootstrap to estimate the confidence interval
+def bootstrap_percentage_change_confidence_interval(data1, data2, n=1000):
+    change_samples = []
+    for _ in range(n):
+        sampled_indices = np.random.choice(
+            range(len(data1)), size=len(data1), replace=True
+        )
+        sampled_data1 = np.array(data1)[sampled_indices]
+        sampled_data2 = np.array(data2)[sampled_indices]
+        change = 100 * (sampled_data2 - sampled_data1) / sampled_data1
+        change_samples.append(np.mean(change))
+    lower, upper = np.percentile(change_samples, [2.5, 97.5])
+    return np.mean(change_samples), lower, upper
+
+
+# DataFrame to store the results
+results_df = pd.DataFrame()
+
+
+# Function to determine the emoji based on change and confidence interval
+def performance_emoji(lower, upper):
+    if upper < -3:
+        return "🟢"  # Emoji for faster performance
+    elif lower > 3:
+        return "🔴"  # Emoji for slower performance
+    else:
+        return "🔵"  # Emoji for insignificant change
+
+
+# Iterate over the models and sizes, perform analysis, and populate the DataFrame
+for model, size in timings_1:
+    model_name = model.__name__
+
+    # Calculate percentage change and confidence interval for init times
+    (
+        init_change,
+        init_lower,
+        init_upper,
+    ) = bootstrap_percentage_change_confidence_interval(
+        timings_1[(model, size)][0], timings_2[(model, size)][0]
+    )
+    init_emoji = performance_emoji(init_lower, init_upper)
+    init_summary = (
+        f"{init_emoji} {init_change:+.1f}% [{init_lower:+.1f}%, {init_upper:+.1f}%]"
+    )
+
+    # Calculate percentage change and confidence interval for run times
+    run_change, run_lower, run_upper = bootstrap_percentage_change_confidence_interval(
+        timings_1[(model, size)][1], timings_2[(model, size)][1]
+    )
+    run_emoji = performance_emoji(run_lower, run_upper)
+    run_summary = (
+        f"{run_emoji} {run_change:+.1f}% [{run_lower:+.1f}%, {run_upper:+.1f}%]"
+    )
+
+    # Append results to DataFrame
+    row = pd.DataFrame(
+        {
+            "Model": [model_name],
+            "Size": [size],
+            "Init time [95% CI]": [init_summary],
+            "Run time [95% CI]": [run_summary],
+        }
+    )
+
+    results_df = pd.concat([results_df, row], ignore_index=True)
+
+# Convert DataFrame to markdown with specified alignments
+markdown_representation = results_df.to_markdown(index=False, tablefmt="github")
+
+# Display the markdown representation
+print(markdown_representation)

benchmarks/WolfSheep/agents.py

@@ -0,0 +1,84 @@
+from Flocking.Flocking import BoidFlockers
+from Schelling.Schelling import SchellingModel
+from WolfSheep.WolfSheep import WolfSheep
+
+configurations = {
+    # Schelling Model Configurations
+    SchellingModel: {
+        "small": {
+            "seeds": 50,
+            "replications": 5,
+            "steps": 20,
+            "parameters": {
+                "height": 40,
+                "width": 40,
+                "homophily": 3,
+                "radius": 1,
+                "density": 0.625,
+            },
+        },
+        "large": {
+            "seeds": 10,
+            "replications": 3,
+            "steps": 10,
+            "parameters": {
+                "height": 100,
+                "width": 100,
+                "homophily": 8,
+                "radius": 2,
+                "density": 0.8,
+            },
+        },
+    },
+    # WolfSheep Model Configurations
+    WolfSheep: {
+        "small": {
+            "seeds": 50,
+            "replications": 5,
+            "steps": 40,
+            "parameters": {
+                "height": 25,
+                "width": 25,
+                "initial_sheep": 60,
+                "initial_wolves": 40,
+                "sheep_reproduce": 0.2,
+                "wolf_reproduce": 0.1,
+                "grass_regrowth_time": 20,
+            },
+        },
+        "large": {
+            "seeds": 10,
+            "replications": 3,
+            "steps": 20,
+            "parameters": {
+                "height": 100,
+                "width": 100,
+                "initial_sheep": 1000,
+                "initial_wolves": 500,
+                "sheep_reproduce": 0.4,
+                "wolf_reproduce": 0.2,
+                "grass_regrowth_time": 10,
+            },
+        },
+    },
+    # BoidFlockers Model Configurations
+    BoidFlockers: {
+        "small": {
+            "seeds": 25,
+            "replications": 3,
+            "steps": 20,
+            "parameters": {"population": 200, "width": 100, "height": 100, "vision": 5},
+        },
+        "large": {
+            "seeds": 10,
+            "replications": 3,
+            "steps": 10,
+            "parameters": {
+                "population": 400,
+                "width": 150,
+                "height": 150,
+                "vision": 15,
+            },
+        },
+    },
+}

benchmarks/WolfSheep/random_walk.py

@@ -0,0 +1,74 @@
+import gc
+import os
+import pickle
+import sys
+import time
+import timeit
+
+from configurations import configurations
+
+
+# Generic function to initialize and run a model
+def run_model(model_class, seed, parameters):
+    start_init = timeit.default_timer()
+    model = model_class(seed=seed, **parameters)
+    #   time.sleep(0.001)
+
+    end_init_start_run = timeit.default_timer()
+
+    for _ in range(config["steps"]):
+        model.step()
+    #       time.sleep(0.0001)
+    end_run = timeit.default_timer()
+
+    return (end_init_start_run - start_init), (end_run - end_init_start_run)
+
+
+# Function to run experiments and save the fastest replication for each seed
+def run_experiments(model_class, config):
+    gc.enable()
+    sys.path.insert(0, os.path.abspath("."))
+
+    init_times = []
+    run_times = []
+    for seed in range(1, config["seeds"] + 1):
+        fastest_init = float("inf")
+        fastest_run = float("inf")
+        for _replication in range(1, config["replications"] + 1):
+            init_time, run_time = run_model(model_class, seed, config["parameters"])
+            if init_time < fastest_init:
+                fastest_init = init_time
+            if run_time < fastest_run:
+                fastest_run = run_time
+        init_times.append(fastest_init)
+        run_times.append(fastest_run)
+
+    return init_times, run_times
+
+
+print(f"{time.strftime('%H:%M:%S', time.localtime())} starting benchmarks.")
+results_dict = {}
+for model, model_config in configurations.items():
+    for size, config in model_config.items():
+        results = run_experiments(model, config)
+
+        mean_init = sum(results[0]) / len(results[0])
+        mean_run = sum(results[1]) / len(results[1])
+
+        print(
+            f"{time.strftime("%H:%M:%S", time.localtime())} {model.__name__:<14} ({size}) timings: Init {mean_init:.5f} s; Run {mean_run:.4f} s"
+        )
+
+        results_dict[model, size] = results
+
+# Change this name to anything you like
+save_name = "timings"
+
+i = 1
+while os.path.exists(f"{save_name}_{i}.pickle"):
+    i += 1
+
+with open(f"{save_name}_{i}.pickle", "wb") as handle:
+    pickle.dump(results_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+print(f"Done benchmarking. Saved results to {save_name}_{i}.pickle.")

benchmarks/compare_timings.py

@@ -7,5 +7,6 @@ coverage:
 
 ignore:
   - "**experimental/"
+  - "benchmarks/**"
 
-comment: off
+comment: off