Measure gas for scenarios (compound-finance#225)

This PR adds gas measurements to the scenario JSON reports.

Scenarios that want their gas measured must now return a transaction receipt of the transaction to be measured.

plugins/scenario/Runner.ts

@@ -79,6 +79,7 @@ export class Runner<T, U> {
     );
     const baseSolutions: Solution<T>[][] = [[identity]];
 
+    let cumulativeGas = 0;
     for (const combo of combos(baseSolutions.concat(solutionChoices))) {
       // create a fresh copy of context that solutions can modify
       let ctx: T = await scenario.forker(context);
@@ -94,7 +95,10 @@ export class Runner<T, U> {
 
       // requirements met, run the property
       try {
-        await scenario.property(await scenario.transformer(ctx), world, ctx);
+        let txnReceipt = await scenario.property(await scenario.transformer(ctx), world, ctx);
+        if (txnReceipt) {
+          cumulativeGas += txnReceipt.cumulativeGasUsed.toNumber();
+        }
         numSolutionSets++;
       } catch (e) {
         // TODO: Include the specific solution (set of states) that failed in the result
@@ -104,13 +108,14 @@ export class Runner<T, U> {
       }
     }
     // Send success result only after all combinations of solutions have passed for this scenario.
-    return this.generateResult(base, scenario, startTime, numSolutionSets);
+    return this.generateResult(base, scenario, startTime, cumulativeGas, numSolutionSets);
   }
 
   private generateResult(
     base: ForkSpec,
     scenario: Scenario<T, U>,
     startTime: number,
+    totalGas: number,
     numSolutionSets: number,
     err?: any
   ): Result {
@@ -126,6 +131,7 @@ export class Runner<T, U> {
       base: base.name,
       file: scenario.file || scenario.name,
       scenario: scenario.name,
+      gasUsed: totalGas / numSolutionSets,
       numSolutionSets,
       elapsed: Date.now() - startTime,
       error: err || null,

plugins/scenario/worker/Parent.ts

@@ -20,6 +20,7 @@ export interface Result {
   base: string;
   file: string;
   scenario: string;
+  gasUsed?: number;
   numSolutionSets?: number;
   elapsed?: number;
   error?: Error;
@@ -148,8 +149,11 @@ export async function runScenario<T, U>(
 
   function mergeResult(index: number, result: Result) {
     pending.delete(key(result.base, result.scenario));
-    // Update the scenario name to include the number of solution sets run.
+    // Update the scenario name to include the number of solution sets run and average gas cost.
     result.scenario += ` [${pluralize(result.numSolutionSets, 'run', 'runs')}]`;
+    if (result.gasUsed) {
+      result.scenario += ` [Avg gas: ${result.gasUsed}]`;
+    }
     results.push(result);
 
     resetStallTimer();

plugins/scenario/worker/Report.ts

@@ -70,6 +70,7 @@ interface JsonTestResult {
   title: string,
   fullTitle: string,
   file: string,
+  gasUsed: number,
   numSolutionSets: number,
   duration: number,
   currentRetry: number,
@@ -107,6 +108,7 @@ async function showJsonReport(results: Result[], jsonOptions: JsonFormatOptions,
       title: result.scenario,
       fullTitle: `${result.base} ${result.scenario}`,
       file: result.file,
+      gasUsed: result.gasUsed ?? 0,
       numSolutionSets: result.numSolutionSets ?? 0,
       duration: result.elapsed || 0,
       currentRetry: 0,

scenario/AllowBySigScenario.ts

@@ -20,7 +20,7 @@ scenario(
       chainId: await world.chainId(),
     });
 
-    await betty.allowBySig({
+    const txn = await betty.allowBySig({
       owner: albert.address,
       manager: betty.address,
       isAllowed: true,
@@ -30,6 +30,8 @@ scenario(
     });
 
     expect(await comet.isAllowed(albert.address, betty.address)).to.be.true;
+
+    return txn; // return txn to measure gas
   }
 );
 

scenario/CometScenario.ts

@@ -6,12 +6,14 @@ scenario('initializes governor correctly', {}, async ({ comet, actors }, world)
   expect(await comet.governor()).to.equal(actors['admin']!.address);
 });
 
-scenario('Comet#allow > allows a user to authorize a manager', {}, async ({ comet, actors }) => {
+scenario('Comet#allow > allows a user to authorize a manager', { upgrade: true }, async ({ comet, actors }) => {
   const { albert, betty } = actors;
 
-  await albert.allow(betty, true);
+  const txn = await albert.allow(betty, true);
 
   expect(await comet.isAllowed(albert.address, betty.address)).to.be.true;
+
+  return txn; // return txn to measure gas
 });
 
 scenario('Comet#allow > allows a user to rescind authorization', {}, async ({ comet, actors }) => {

scenario/PauseGuardianScenario.ts

@@ -16,7 +16,7 @@ scenario(
     expect(await comet.isBuyPaused()).to.be.false;
 
     const { admin } = actors;
-    await admin.pause({
+    const txn = await admin.pause({
       supplyPaused: true,
       transferPaused: true,
       withdrawPaused: true,
@@ -29,6 +29,8 @@ scenario(
     expect(await comet.isWithdrawPaused()).to.be.true;
     expect(await comet.isAbsorbPaused()).to.be.true;
     expect(await comet.isBuyPaused()).to.be.true;
+
+    return txn; // return txn to measure gas
   }
 );
 

scenario/WithdrawReservesScenario.ts

@@ -16,10 +16,12 @@ scenario(
 
     expect(await baseToken.balanceOf(comet.address)).to.equal(100n);
 
-    await admin.withdrawReserves(albert, 10);
+    const txn = await admin.withdrawReserves(albert, 10);
 
     expect(await baseToken.balanceOf(comet.address)).to.equal(90n);
     expect(await baseToken.balanceOf(albert.address)).to.equal(10n);
+
+    return txn; // return txn to measure gas
   }
 );
 

scenario/context/CometActor.ts

@@ -1,5 +1,5 @@
 import { SignerWithAddress } from '@nomiclabs/hardhat-ethers/signers';
-import { BigNumberish, Signature, ethers } from 'ethers';
+import { BigNumberish, Signature, ethers, ContractReceipt } from 'ethers';
 import { CometContext } from './CometContext';
 import { AddressLike, resolveAddress } from './Address';
 
@@ -54,9 +54,9 @@ export default class CometActor {
     await tx.wait();
   }
 
-  async allow(manager: CometActor, isAllowed: boolean) {
+  async allow(manager: CometActor, isAllowed: boolean): Promise<ContractReceipt> {
     let comet = await this.context.getComet();
-    await (await comet.connect(this.signer).allow(manager.address, isAllowed)).wait();
+    return await (await comet.connect(this.signer).allow(manager.address, isAllowed)).wait();
   }
 
   async pause({
@@ -65,23 +65,23 @@ export default class CometActor {
     withdrawPaused = false,
     absorbPaused = false,
     buyPaused = false,
-  }) {
+  }): Promise<ContractReceipt> {
     let comet = await this.context.getComet();
-    await (
+    return await (
       await comet
         .connect(this.signer)
         .pause(supplyPaused, transferPaused, withdrawPaused, absorbPaused, buyPaused)
     ).wait();
   }
 
-  async transferAsset({ dst, asset, amount }) {
+  async transferAsset({ dst, asset, amount }): Promise<ContractReceipt> {
     let comet = await this.context.getComet();
-    await (await comet.connect(this.signer).transferAsset(dst, asset, amount)).wait();
+    return await (await comet.connect(this.signer).transferAsset(dst, asset, amount)).wait();
   }
 
-  async transferAssetFrom({ src, dst, asset, amount }) {
+  async transferAssetFrom({ src, dst, asset, amount }): Promise<ContractReceipt> {
     let comet = await this.context.getComet();
-    await (await comet.connect(this.signer).transferAssetFrom(src, dst, asset, amount)).wait();
+    return await (await comet.connect(this.signer).transferAssetFrom(src, dst, asset, amount)).wait();
   }
 
   async signAuthorization({
@@ -129,19 +129,19 @@ export default class CometActor {
     nonce: BigNumberish;
     expiry: number;
     signature: Signature;
-  }) {
+  }): Promise<ContractReceipt> {
     let comet = await this.context.getComet();
-    await comet
+    return await (await comet
       .connect(this.signer)
-      .allowBySig(owner, manager, isAllowed, nonce, expiry, signature.v, signature.r, signature.s);
+      .allowBySig(owner, manager, isAllowed, nonce, expiry, signature.v, signature.r, signature.s)).wait();
   }
 
   async show() {
     return console.log(`Actor#${this.name}{${JSON.stringify(this.info)}}`);
   }
 
-  async withdrawReserves(to: CometActor, amount: number) {
+  async withdrawReserves(to: CometActor, amount: number): Promise<ContractReceipt> {
     let comet = await this.context.getComet();
-    await (await comet.connect(this.signer).withdrawReserves(to.address, amount)).wait();
+    return await (await comet.connect(this.signer).withdrawReserves(to.address, amount)).wait();
   }
 }

tasks/scenario/task.ts

@@ -27,7 +27,7 @@ task('scenario', 'Runs scenario tests')
   .addOptionalParam('bases', 'Bases to run on [defaults to all]')
   .addFlag('noSpider', 'skip spider')
   .addFlag('sync', 'run synchronously')
-  .addOptionalParam('stall', 'milliseconds to wait until we fail for stalling', 60000, types.int)
+  .addOptionalParam('stall', 'milliseconds to wait until we fail for stalling', 120000, types.int)
   .addOptionalParam('workers', 'count of workers', 6, types.int)
   .setAction(async (taskArgs, env: HardhatRuntimeEnvironment) => {
     let bases: ForkSpec[] = getBasesFromTaskArgs(taskArgs.bases, env);