Feature/multiple txs by sequencer (0xPolygonHermez#1108)

* implemented multiple txs processing by the sequencer * implemented multiple txs processing by the sequencer * fixed array passing when getting top tx * adding logs and fixed bugs * refactored functions related to zk counters calculation * fixed pr comments * Discard batch by OOC (0xPolygonHermez#1110) * Discard batch by OOC * Fix isBatchOutOfCounters * Remove not needed check in test * Improve OOC detection check * Change field name to UnprocesedBatch * Update prover docker image * Update prover docker image * Update prover docker image * Update prover docker image * Update prover docker image * get rid of updateTxStateInPool function in the sequencer and changes tx state -> tx status * fixed pool_test.go * Reprocess discarded batch by OOC error in batchbuilder (0xPolygonHermez#1114) * Reprocess discarded batch * Rename function * Bump github.com/urfave/cli/v2 from 2.11.2 to 2.14.0 (0xPolygonHermez#1113) Bumps [github.com/urfave/cli/v2](https://github.com/urfave/cli) from 2.11.2 to 2.14.0. - [Release notes](https://github.com/urfave/cli/releases) - [Changelog](https://github.com/urfave/cli/blob/main/docs/CHANGELOG.md) - [Commits](https://github.com/urfave/cli/commits) --- updated-dependencies: - dependency-name: github.com/urfave/cli/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <[email protected]> Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Use CONFIG_MODE=test for E2E test (0xPolygonHermez#1117) * Use CONFIG_MODE=test for E2E test * Add -v flag to tests * Include logic in a loop * Correct comments * Change processTxResponse struct Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Arnau Bennassar <[email protected]> * fixed slow query * fixed test * Fix/is processed tx (0xPolygonHermez#1138) * fix isProcessed * WIP * WIP * WIP * sequencer refactor * pass lint * fix e2e * increase sleep time for test-full-non-e2e * change prover version * fixed pool_test.go * small clean up Co-authored-by: Toni Ramírez <[email protected]> Co-authored-by: Mikelle <[email protected]> Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Toni Ramírez <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Arnau Bennassar <[email protected]> Co-authored-by: Toni Ramírez <[email protected]>
satman81 · Sep 13, 2022 · 69fd553 · 69fd553
1 parent 0d25458
commit 69fd553
Show file tree

Hide file tree

Showing 29 changed files with 608 additions and 428 deletions.
diff --git a/Makefile b/Makefile
@@ -117,7 +117,7 @@ test-full-non-e2e: build-docker compile-scs ## Runs non-e2e tests checking race
 	$(RUNDB); sleep 7
 	$(RUNZKPROVER)
 	$(RUNZKPROVERMOCK)
-	sleep 5
+	sleep 15
 	docker logs $(DOCKERCOMPOSEZKPROVER)
 	trap '$(STOPDB) && $(STOPZKPROVER) && $(STOPZKPROVERMOCK)' EXIT; MallocNanoZone=0 go test -short -race -p 1 -timeout 60s ./...
 

diff --git a/config/config.debug.toml b/config/config.debug.toml
@@ -10,7 +10,7 @@ Password = "test_password"
 Name = "test_db"
 Host = "localhost"
 Port = "5432"
-EnableLog = true
+EnableLog = false
 MaxConns = 10
 
 [Etherman]
@@ -27,7 +27,7 @@ FrequencyForResendingFailedVerifyBatch = "1s"
 [RPC]
 Host = "0.0.0.0"
 Port = 8123
-MaxRequestsPerIPAndSecond = 100
+MaxRequestsPerIPAndSecond = 10000
 SequencerNodeURI = ""
 BroadcastURI = "127.0.0.1:61090"
 DefaultSenderAddress = "0x1111111111111111111111111111111111111111"

diff --git a/config/config.local.toml b/config/config.local.toml
@@ -75,7 +75,7 @@ Port = 50060
 StoreBackend = "PostgreSQL"
 
 [MTClient]
-URI  = "zkevm-prover:50061"
+URI = "zkevm-prover:50061"
 
 [Executor]
 URI = "zkevm-prover:50071"

diff --git a/db/migrations/0001.sql b/db/migrations/0001.sql
@@ -6,107 +6,115 @@ DROP SCHEMA IF EXISTS rpc CASCADE;
 -- +migrate Up
 CREATE SCHEMA state;
 
-CREATE TABLE state.block ( --L1 block
-    block_num BIGINT PRIMARY KEY,
-    block_hash VARCHAR NOT NULL,
+CREATE TABLE state.block
+( --L1 block
+    block_num   BIGINT PRIMARY KEY,
+    block_hash  VARCHAR                  NOT NULL,
     parent_hash VARCHAR,
     received_at TIMESTAMP WITH TIME ZONE NOT NULL
 );
 
-CREATE TABLE state.batch (  --batch abstraction: will be created through trusted state
-    batch_num BIGINT PRIMARY KEY,
+CREATE TABLE state.batch
+( --batch abstraction: will be created through trusted state
+    batch_num        BIGINT PRIMARY KEY,
     global_exit_root VARCHAR,
-    local_exit_root VARCHAR,
-    state_root VARCHAR,
-    timestamp TIMESTAMP,
-    coinbase VARCHAR,
-    raw_txs_data BYTEA
+    local_exit_root  VARCHAR,
+    state_root       VARCHAR,
+    timestamp        TIMESTAMP,
+    coinbase         VARCHAR,
+    raw_txs_data     BYTEA
 );
 
-CREATE TABLE state.virtual_batch (  --virtual state
+CREATE TABLE state.virtual_batch
+( --virtual state
     batch_num BIGINT PRIMARY KEY REFERENCES state.batch (batch_num) ON DELETE CASCADE,
-    tx_hash VARCHAR,
-    coinbase VARCHAR,
+    tx_hash   VARCHAR,
+    coinbase  VARCHAR,
     block_num BIGINT NOT NULL REFERENCES state.block (block_num) ON DELETE CASCADE
 );
 
-CREATE TABLE state.verified_batch (  --consolidated state
-    batch_num BIGINT PRIMARY KEY REFERENCES state.virtual_batch (batch_num) ON DELETE CASCADE,
-    tx_hash VARCHAR,
+CREATE TABLE state.verified_batch
+( --consolidated state
+    batch_num  BIGINT PRIMARY KEY REFERENCES state.virtual_batch (batch_num) ON DELETE CASCADE,
+    tx_hash    VARCHAR,
     aggregator VARCHAR,
-    block_num BIGINT NOT NULL REFERENCES state.block (block_num) ON DELETE CASCADE
+    block_num  BIGINT NOT NULL REFERENCES state.block (block_num) ON DELETE CASCADE
 );
 
-CREATE TABLE state.forced_batch (
+CREATE TABLE state.forced_batch
+(
     forced_batch_num BIGINT PRIMARY KEY,
     global_exit_root VARCHAR,
-    timestamp TIMESTAMP,
-    raw_txs_data VARCHAR,
-    coinbase VARCHAR,
-    batch_num BIGINT, -- It can be null if the batch state is not trusted
-    block_num BIGINT NOT NULL REFERENCES state.block (block_num) ON DELETE CASCADE
+    timestamp        TIMESTAMP,
+    raw_txs_data     VARCHAR,
+    coinbase         VARCHAR,
+    batch_num        BIGINT, -- It can be null if the batch state is not trusted
+    block_num        BIGINT NOT NULL REFERENCES state.block (block_num) ON DELETE CASCADE
 );
 
-CREATE TABLE state.l2block (
-    block_num BIGINT PRIMARY KEY,
-    block_hash VARCHAR NOT NULL,
-    header jsonb,
-    uncles jsonb,
+CREATE TABLE state.l2block
+(
+    block_num   BIGINT PRIMARY KEY,
+    block_hash  VARCHAR                  NOT NULL,
+    header      jsonb,
+    uncles      jsonb,
     parent_hash VARCHAR,
-    state_root VARCHAR,
+    state_root  VARCHAR,
     received_at TIMESTAMP WITH TIME ZONE NOT NULL,
-    batch_num BIGINT NOT NULL REFERENCES state.batch (batch_num) ON DELETE CASCADE
+    batch_num   BIGINT                   NOT NULL REFERENCES state.batch (batch_num) ON DELETE CASCADE
 );
 
-CREATE TABLE state.transaction (
-    hash VARCHAR PRIMARY KEY,
-    encoded VARCHAR NOT NULL,
-    decoded jsonb,
-    l2_block_num BIGINT NOT NULL REFERENCES state.l2block (block_num) ON DELETE CASCADE
+CREATE TABLE state.transaction
+(
+    hash         VARCHAR PRIMARY KEY,
+    encoded      VARCHAR NOT NULL,
+    decoded      jsonb,
+    l2_block_num BIGINT  NOT NULL REFERENCES state.l2block (block_num) ON DELETE CASCADE
 );
 
 CREATE TABLE state.exit_root
 (
-    block_num               BIGINT NOT NULL REFERENCES state.block (block_num) ON DELETE CASCADE,
-    global_exit_root_num    BIGINT PRIMARY KEY,
-    mainnet_exit_root       BYTEA,
-    rollup_exit_root        BYTEA,
-    global_exit_root        BYTEA
+    block_num            BIGINT NOT NULL REFERENCES state.block (block_num) ON DELETE CASCADE,
+    global_exit_root_num BIGINT PRIMARY KEY,
+    mainnet_exit_root    BYTEA,
+    rollup_exit_root     BYTEA,
+    global_exit_root     BYTEA
 );
 
 CREATE TABLE state.sync_info
 (
-    last_batch_num_seen BIGINT,
+    last_batch_num_seen         BIGINT,
     last_batch_num_consolidated BIGINT,
-    init_sync_batch BIGINT
+    init_sync_batch             BIGINT
 );
 
 -- Insert default values into sync_info table
-INSERT INTO state.sync_info (last_batch_num_seen, last_batch_num_consolidated, init_sync_batch)VALUES (0, 0, 0);
+INSERT INTO state.sync_info (last_batch_num_seen, last_batch_num_consolidated, init_sync_batch)
+VALUES (0, 0, 0);
 
 CREATE TABLE state.receipt
 (
-    tx_hash VARCHAR NOT NULL PRIMARY KEY REFERENCES state.transaction (hash) ON DELETE CASCADE,
-    type integer,
-    post_state BYTEA,
-    status BIGINT,
+    tx_hash             VARCHAR NOT NULL PRIMARY KEY REFERENCES state.transaction (hash) ON DELETE CASCADE,
+    type                integer,
+    post_state          BYTEA,
+    status              BIGINT,
     cumulative_gas_used BIGINT,
-    gas_used BIGINT,
-    block_num BIGINT NOT NULL REFERENCES state.l2block (block_num) ON DELETE CASCADE,
-    tx_index integer,
-    contract_address VARCHAR
+    gas_used            BIGINT,
+    block_num           BIGINT  NOT NULL REFERENCES state.l2block (block_num) ON DELETE CASCADE,
+    tx_index            integer,
+    contract_address    VARCHAR
 );
 
 CREATE TABLE state.log
 (
-    tx_hash VARCHAR NOT NULL REFERENCES state.transaction (hash) ON DELETE CASCADE,
+    tx_hash   VARCHAR NOT NULL REFERENCES state.transaction (hash) ON DELETE CASCADE,
     log_index integer,
-    address VARCHAR NOT NULL,
-    data VARCHAR,
-    topic0 VARCHAR NOT NULL,
-    topic1 VARCHAR,
-    topic2 VARCHAR,
-    topic3 VARCHAR,
+    address   VARCHAR NOT NULL,
+    data      VARCHAR,
+    topic0    VARCHAR NOT NULL,
+    topic1    VARCHAR,
+    topic2    VARCHAR,
+    topic3    VARCHAR,
     PRIMARY KEY (tx_hash, log_index)
 );
 
@@ -117,7 +125,7 @@ CREATE TABLE pool.txs
     hash                   VARCHAR PRIMARY KEY,
     encoded                VARCHAR,
     decoded                jsonb,
-    state                  varchar(15),
+    status                 varchar(15),
     gas_price              DECIMAL(78, 0),
     nonce                  DECIMAL(78, 0),
     is_claims              BOOLEAN,
@@ -130,10 +138,10 @@ CREATE TABLE pool.txs
     used_binaries          INTEGER,
     used_steps             INTEGER,
     received_at            TIMESTAMP WITH TIME ZONE NOT NULL,
-    from_address           varchar NOT NULL
+    from_address           varchar                  NOT NULL
 );
 
-CREATE INDEX idx_state_gas_price_nonce ON pool.txs (state, gas_price, nonce);
+CREATE INDEX idx_state_gas_price_nonce ON pool.txs (status, gas_price, nonce);
 
 CREATE TABLE pool.gas_price
 (
@@ -144,10 +152,10 @@ CREATE TABLE pool.gas_price
 
 CREATE SCHEMA rpc
 
-CREATE TABLE rpc.filters
-(
-    id          SERIAL PRIMARY KEY,
-    filter_type VARCHAR(15) NOT NULL,
-    parameters  JSONB       NOT NULL,
-    last_poll   TIMESTAMP   NOT NULL
-);
+    CREATE TABLE rpc.filters
+    (
+        id          SERIAL PRIMARY KEY,
+        filter_type VARCHAR(15) NOT NULL,
+        parameters  JSONB       NOT NULL,
+        last_poll   TIMESTAMP   NOT NULL
+    );
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -143,7 +143,7 @@ services:
       - MIX_ENV=prod
     command:
       ["/bin/sh", "-c", "mix do ecto.create, ecto.migrate; mix phx.server"]
-  
+
   zkevm-explorer-l1-db:
     container_name: zkevm-explorer-l1-db
     image: postgres
@@ -206,7 +206,7 @@ services:
     command: ["postgres", "-N", "500"]
 
   zkevm-mock-l1-network:
-    container_name: zkevm-local-l1-network
+    container_name: zkevm-mock-l1-network
     image: hermeznetwork/geth-zkevm-contracts@sha256:b88d03a42ef4975d0b92c0dd192e1bf4d673fc4b761aaf4bcfdd606989c1b006
     ports:
       - 8545:8545

diff --git a/etherman/etherman_test.go b/etherman/etherman_test.go
@@ -294,10 +294,9 @@ func TestSendSequences(t *testing.T) {
 
 	tx1 := types.NewTransaction(uint64(0), common.Address{}, big.NewInt(10), uint64(1), big.NewInt(10), []byte{})
 	sequence := ethmanTypes.Sequence{
-		GlobalExitRoot:  ger,
-		Timestamp:       int64(currentBlock.Time() - 1),
-		ForceBatchesNum: 0,
-		Txs:             []types.Transaction{*tx1},
+		GlobalExitRoot: ger,
+		Timestamp:      int64(currentBlock.Time() - 1),
+		Txs:            []types.Transaction{*tx1},
 	}
 	tx, err := etherman.sequenceBatches(etherman.auth, []ethmanTypes.Sequence{sequence})
 	require.NoError(t, err)

diff --git a/etherman/types/sequence.go b/etherman/types/sequence.go
@@ -3,19 +3,16 @@ package types
 import (
 	"reflect"
 
-	"github.com/0xPolygonHermez/zkevm-node/pool"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
 // Sequence represents an operation sent to the PoE smart contract to be
 // processed.
 type Sequence struct {
-	GlobalExitRoot  common.Hash
-	Timestamp       int64
-	ForceBatchesNum uint64
-	Txs             []types.Transaction
-	pool.ZkCounters
+	GlobalExitRoot, StateRoot, LocalExitRoot common.Hash
+	Timestamp                                int64
+	Txs                                      []types.Transaction
 }
 
 // IsEmpty checks is sequence struct is empty

diff --git a/go.mod b/go.mod
@@ -14,6 +14,7 @@ require (
 	github.com/imdario/mergo v0.3.13
 	github.com/jackc/pgconn v1.13.0
 	github.com/jackc/pgx/v4 v4.17.2
+	github.com/lib/pq v1.10.2
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/rubenv/sql-migrate v0.0.0-20211023115951-9f02b1e13857
 	github.com/spf13/afero v1.9.2

diff --git a/pool/interfaces.go b/pool/interfaces.go
@@ -11,19 +11,19 @@ import (
 
 type storage interface {
 	AddTx(ctx context.Context, tx Transaction) error
-	CountTransactionsByState(ctx context.Context, state TxState) (uint64, error)
+	CountTransactionsByStatus(ctx context.Context, status TxStatus) (uint64, error)
 	DeleteTxsByHashes(ctx context.Context, hashes []common.Hash) error
 	GetGasPrice(ctx context.Context) (uint64, error)
 	GetNonce(ctx context.Context, address common.Address) (uint64, error)
 	GetPendingTxHashesSince(ctx context.Context, since time.Time) ([]common.Hash, error)
-	GetTopPendingTxByProfitabilityAndZkCounters(ctx context.Context, maxZkCounters ZkCounters) (*Transaction, error)
 	GetTxsByFromAndNonce(ctx context.Context, from common.Address, nonce uint64) ([]Transaction, error)
-	GetTxsByState(ctx context.Context, state TxState, isClaims bool, limit uint64) ([]Transaction, error)
+	GetTxsByStatus(ctx context.Context, state TxStatus, isClaims bool, limit uint64) ([]Transaction, error)
 	IsTxPending(ctx context.Context, hash common.Hash) (bool, error)
 	MarkReorgedTxsAsPending(ctx context.Context) error
 	SetGasPrice(ctx context.Context, gasPrice uint64) error
-	UpdateTxsState(ctx context.Context, hashes []common.Hash, newState TxState) error
-	UpdateTxState(ctx context.Context, hash common.Hash, newState TxState) error
+	UpdateTxsStatus(ctx context.Context, hashes []string, newStatus TxStatus) error
+	UpdateTxStatus(ctx context.Context, hash common.Hash, newStatus TxStatus) error
+	GetPendingTxsWithLowestNonce(ctx context.Context, limit uint64) ([]*Transaction, error)
 }
 
 type stateInterface interface {