Updated get_validators_max_index and added is_constant flag in calling

tester contract

1. Updated `get_validators_max_index` function in validator manager
contract to prevent inconsistent shard_list. Note that the deposit would
be activated since next shuffling cycle.
2. Separate some basic utils function from validator_manager_utils.py ->
contract_utils.py
3. Rename `SHUFFLING_CYCLE`  to `SHUFFLING_CYCLE_LENGTH`
4. Set SHUFFLING_CYCLE_LENGTH from 2500 to 25 for now (easier for
testing before EIP96 is finished)
5. Updated tester function call: added `is_constant` flag so it can be
used for calling constant function and it won’t change the head state

docs/doc.md

@@ -22,7 +22,7 @@ We assume that at address `VALIDATOR_MANAGER_ADDRESS` (on the existing "main sha
 -   `SIG_GASLIMIT`: 40000
 -   `COLLATOR_REWARD`: 0.002
 -   `PERIOD_LENGTH`: 5 blocks
--   `SHUFFLING_CYCLE`: 2500 blocks
+-   `SHUFFLING_CYCLE_LENGTH`: 2500 blocks
 
 ### Specification
 
@@ -90,13 +90,13 @@ The transaction has an additional side effect of saving a record in `USED_RECEIP
 
 ### Details of `sample`
 
-The `sample` function should be coded in such a way that any given validator randomly gets allocated to some number of shards every `SHUFFLING_CYCLE`, where the expected number of shards is proportional to the validator's balance. During that cycle, `sample(shard_id)` can only return that validator if the `shard_id` is one of the shards that they were assigned to. The purpose of this is to give validators time to download the state of the specific shards that they are allocated to.
+The `sample` function should be coded in such a way that any given validator randomly gets allocated to some number of shards every `SHUFFLING_CYCLE_LENGTH`, where the expected number of shards is proportional to the validator's balance. During that cycle, `sample(shard_id)` can only return that validator if the `shard_id` is one of the shards that they were assigned to. The purpose of this is to give validators time to download the state of the specific shards that they are allocated to.
 
 Here is one possible implementation of `sample`, assuming for simplicity of illustration that all validators have the same deposit size:
 
     def sample(shard_id: num) -> address:
-        cycle = floor(block.number / SHUFFLING_CYCLE)
-        cycle_seed = blockhash(cycle * SHUFFLING_CYCLE)
+        cycle = floor(block.number / SHUFFLING_CYCLE_LENGTH)
+        cycle_seed = blockhash(cycle * SHUFFLING_CYCLE_LENGTH)
         seed = blockhash(block.number - (block.number % PERIOD_LENGTH))
         index_in_subset = num256_mod(as_num256(sha3(concat(seed, as_bytes32(shard_id)))),
                                      100)
@@ -110,7 +110,7 @@ This picks out 100 validators for each shard during each cycle, and then during
 
 We generally expect collation headers to be produced and propagated as follows.
 
-* Every time a new `SHUFFLING_CYCLE` starts, every validator computes the set of 100 validators for every shard that they were assigned to, and sees which shards they are eligible to validate in. The validator then downloads the state for that shard (using fast sync)
+* Every time a new `SHUFFLING_CYCLE_LENGTH` starts, every validator computes the set of 100 validators for every shard that they were assigned to, and sees which shards they are eligible to validate in. The validator then downloads the state for that shard (using fast sync)
 * The validator keeps track of the head of the chain for all shards they are currently assigned to. It is each validator's responsibility to reject invalid or unavailable collations, and refuse to build on such blocks, even if those blocks get accepted by the main chain validator manager contract.
 * If a validator is currently eligible to validate in some shard `i`, they download the full collation association with any collation header that is included into block headers for shard `i`.
 * When on the current global main chain a new period starts, the validator calls `sample(i)` to determine if they are eligible to create a collation; if they are, then they do so.

sharding/collator.py

@@ -5,10 +5,8 @@
 from ethereum.common import mk_block_from_prevstate
 
 from sharding import state_transition
-from sharding.validator_manager_utils import (
-    sign,
-    call_valmgr,
-)
+from sharding.contract_utils import sign
+from sharding.validator_manager_utils import call_valmgr
 from sharding.receipt_consuming_tx_utils import apply_shard_transaction
 
 log = get_logger('sharding.collator')

sharding/config.py

@@ -20,7 +20,7 @@
 sharding_config['COLLATOR_REWARD'] = 0.002 * utils.denoms.ether
 sharding_config['SIG_GASLIMIT'] = 40000
 sharding_config['PERIOD_LENGTH'] = 5                 # blocks
-sharding_config['SHUFFLING_CYCLE'] = 2500            # blocks
+sharding_config['SHUFFLING_CYCLE_LENGTH'] = 25              # blocks, TODO: 25 is for testing, the reasonable number is 2500
 sharding_config['DEPOSIT_SIZE'] = 10 ** 20
 sharding_config['CONTRACT_CALL_GAS'] = {
     'VALIDATOR_MANAGER': defaultdict(lambda: 200000, {

sharding/contract_utils.py

@@ -0,0 +1,92 @@
+import rlp
+
+from ethereum import (
+    utils,
+    vm,
+)
+from ethereum.messages import apply_message
+from ethereum.transactions import Transaction
+
+
+STARTGAS = 3141592   # TODO: use config
+GASPRICE = 1         # TODO: use config
+
+
+class MessageFailed(Exception):
+    pass
+
+
+def sign(msg_hash, privkey):
+    v, r, s = utils.ecsign(msg_hash, privkey)
+    signature = utils.encode_int32(v) + utils.encode_int32(r) + utils.encode_int32(s)
+    return signature
+
+
+def get_tx_rawhash(tx, network_id=None):
+    """Get a tx's rawhash.
+       Copied from ethereum.transactions.Transaction.sign
+    """
+    if network_id is None:
+        rawhash = utils.sha3(rlp.encode(tx, Transaction.exclude(['v', 'r', 's'])))
+    else:
+        assert 1 <= network_id < 2**63 - 18
+        rlpdata = rlp.encode(rlp.infer_sedes(tx).serialize(tx)[:-3] + [network_id, b'', b''])
+        rawhash = utils.sha3(rlpdata)
+    return rawhash
+
+
+def extract_sender_from_tx(tx):
+    tx_rawhash = get_tx_rawhash(tx)
+    return utils.sha3(
+        utils.ecrecover_to_pub(tx_rawhash, tx.v, tx.r, tx.s)
+    )[-20:]
+
+
+def call_msg(state, ct, func, args, sender_addr, to, value=0, startgas=STARTGAS):
+    abidata = vm.CallData([utils.safe_ord(x) for x in ct.encode_function_call(func, args)])
+    msg = vm.Message(sender_addr, to, value, startgas, abidata)
+    result = apply_message(state, msg)
+    if result is None:
+        raise MessageFailed("Msg failed")
+    if result is False:
+        return result
+    if result == b'':
+        return None
+    o = ct.decode(func, result)
+    return o[0] if len(o) == 1 else o
+
+
+def call_contract_constantly(state, ct, contract_addr, func, args, value=0, startgas=200000, sender_addr=b'\x00' * 20):
+    return call_msg(
+        state.ephemeral_clone(), ct, func, args,
+        sender_addr, contract_addr, value, startgas
+    )
+
+
+def call_contract_inconstantly(state, ct, contract_addr, func, args, value=0, startgas=200000, sender_addr=b'\x00' * 20):
+    result = call_msg(
+        state, ct, func, args, sender_addr, contract_addr, value, startgas
+    )
+    state.commit()
+    return result
+
+
+def call_tx(state, ct, func, args, sender, to, value=0, startgas=STARTGAS, gasprice=GASPRICE, nonce=None):
+    # Transaction(nonce, gasprice, startgas, to, value, data, v=0, r=0, s=0)
+    tx = Transaction(
+        state.get_nonce(utils.privtoaddr(sender)) if nonce is None else nonce,
+        gasprice, startgas, to, value,
+        ct.encode_function_call(func, args)
+    ).sign(sender)
+    return tx
+
+
+def create_contract_tx(state, sender_privkey, bytecode, startgas=STARTGAS):
+    """Generate create contract transaction
+    """
+    tx = Transaction(
+        state.get_nonce(utils.privtoaddr(sender_privkey)),
+        GASPRICE, startgas, to=b'', value=0,
+        data=bytecode
+    ).sign(sender_privkey)
+    return tx

sharding/contracts/validator_manager.v.py

@@ -6,6 +6,8 @@
     validation_code_addr: address,
     # Addess to withdraw to
     return_addr: address,
+    # The cycle number which the validator would be included after
+    cycle: num,
 }[num])
 
 collation_headers: public({
@@ -65,7 +67,7 @@ def __init__():
     self.empty_slots_stack_top = 0
     # 10 ** 20 wei = 100 ETH
     self.deposit_size = 100000000000000000000
-    self.shuffling_cycle_length = 2500
+    self.shuffling_cycle_length = 25
     self.sig_gas_limit = 400000
     self.period_length = 5
     self.num_validators_per_cycle = 100
@@ -91,22 +93,37 @@ def stack_pop() -> num:
 
 
 def get_validators_max_index() -> num:
-    return self.num_validators + self.empty_slots_stack_top
+    zero_addr = 0x0000000000000000000000000000000000000000
+    activate_validator_num = 0
+    current_cycle = floor(decimal(block.number / self.shuffling_cycle_length))
+    all_validator_slots_num = self.num_validators + self.empty_slots_stack_top
+
+    # TODO: any better way to iterate the mapping?
+    for i in range(1024):
+        if i >= all_validator_slots_num:
+            break
+        if (self.validators[i].validation_code_addr != zero_addr and
+            self.validators[i].cycle <= current_cycle):
+            activate_validator_num += 1
+    return activate_validator_num + self.empty_slots_stack_top
 
 
 @payable
 def deposit(validation_code_addr: address, return_addr: address) -> num:
     assert not self.is_valcode_deposited[validation_code_addr]
     assert msg.value == self.deposit_size
     # find the empty slot index in validators set
+    next_cycle = 0
     if not self.is_stack_empty():
         index = self.stack_pop()
     else:
         index = self.num_validators
+        next_cycle = floor(decimal(block.number / self.shuffling_cycle_length)) + 1
     self.validators[index] = {
         deposit: msg.value,
         validation_code_addr: validation_code_addr,
-        return_addr: return_addr
+        return_addr: return_addr,
+        cycle: next_cycle
     }
     self.num_validators += 1
     self.is_valcode_deposited[validation_code_addr] = True
@@ -122,7 +139,8 @@ def withdraw(validator_index: num, sig: bytes <= 1000) -> bool:
         self.validators[validator_index] = {
             deposit: 0,
             validation_code_addr: None,
-            return_addr: None
+            return_addr: None,
+            cycle: 0
         }
         self.stack_push(validator_index)
         self.num_validators -= 1
@@ -131,7 +149,6 @@ def withdraw(validator_index: num, sig: bytes <= 1000) -> bool:
 
 @constant
 def sample(shard_id: num) -> address:
-
     cycle = floor(decimal(block.number / self.shuffling_cycle_length))
     cycle_start_block_number = cycle * self.shuffling_cycle_length - 1
     if cycle_start_block_number < 0:
@@ -146,9 +163,10 @@ def sample(shard_id: num) -> address:
                                  as_num256(self.num_validators_per_cycle))
     validator_index = num256_mod(as_num256(sha3(concat(cycle_seed, as_bytes32(shard_id), as_bytes32(index_in_subset)))),
                                  as_num256(self.get_validators_max_index()))
-    addr = self.validators[as_num128(validator_index)].validation_code_addr
-
-    return addr
+    if self.validators[as_num128(validator_index)].cycle > cycle:
+        return 0x0000000000000000000000000000000000000000
+    else:
+        return self.validators[as_num128(validator_index)].validation_code_addr
 
 
 # Get all possible shard ids that the given valcode_addr
@@ -161,6 +179,7 @@ def get_shard_list(valcode_addr: address) -> bool[100]:
     if cycle_start_block_number < 0:
         cycle_start_block_number = 0
     cycle_seed = blockhash(cycle_start_block_number)
+    validators_max_index = self.get_validators_max_index()
     if self.num_validators != 0:
         for shard_id in range(100):
             shard_list[shard_id] = False
@@ -176,10 +195,9 @@ def get_shard_list(valcode_addr: address) -> bool[100]:
                                 as_bytes32(possible_index_in_subset))
                             )
                     ),
-                    as_num256(self.get_validators_max_index())
+                    as_num256(validators_max_index)
                 )
-                addr = self.validators[as_num128(validator_index)].validation_code_addr
-                if addr == valcode_addr:
+                if valcode_addr == self.validators[as_num128(validator_index)].validation_code_addr:
                     shard_list[shard_id] = True
                     break
     return shard_list

sharding/receipt_consuming_tx_utils.py

@@ -13,14 +13,13 @@
 )
 from ethereum.slogging import get_logger
 from ethereum.transactions import Transaction
-
+from sharding.contract_utils import call_contract_inconstantly
 from sharding.used_receipt_store_utils import (
     call_urs,
     get_urs_ct,
     get_urs_contract,
 )
 from sharding.validator_manager_utils import (
-    call_contract_inconstantly,
     call_valmgr,
 )
 

sharding/tests/test_collator.py

@@ -8,6 +8,7 @@
 
 from sharding import collator
 from sharding.tools import tester
+from sharding.config import sharding_config
 
 log = get_logger('test.collator')
 log.setLevel(logging.DEBUG)
@@ -24,7 +25,7 @@ def chain(shard_id, k0_deposit=True):
     if k0_deposit:
         # deposit
         c.sharding_deposit(privkey, valcode_addr)
-        c.mine(1)
+        c.mine(sharding_config['SHUFFLING_CYCLE_LENGTH'])
     c.add_test_shard(shard_id)
     return c
 

sharding/tests/test_main_chain.py

@@ -7,6 +7,7 @@
 from sharding.tools import tester
 from sharding.shard_chain import ShardChain
 from sharding.collation import Collation
+from sharding.config import sharding_config
 
 log = get_logger('test.shard_chain')
 log.setLevel(logging.DEBUG)
@@ -121,7 +122,7 @@ def chain(shard_id, k0_deposit=True):
     if k0_deposit:
         # deposit
         c.sharding_deposit(privkey, valcode_addr)
-        c.mine(1)
+        c.mine(sharding_config['SHUFFLING_CYCLE_LENGTH'])
     c.add_test_shard(shard_id)
     return c
 
@@ -165,7 +166,7 @@ def test_longest_chain_rule():
     log.info('[block 2\'] CURRENT BLOCK HEAD:{}'.format(encode_hex(t.chain.head_hash)))
     log.info('[block 2\'] CURRENT SHARD HEAD:{}'.format(encode_hex(t.chain.shards[shard_id].head_hash)))
     assert t.chain.shards[shard_id].get_score(t.chain.shards[shard_id].head) == 1
-    assert t.chain.get_score(t.chain.head) == 13
+    assert t.chain.get_score(t.chain.head) == 37
 
     # [block 3']: includes collation B -> C
     # Clear tester
@@ -180,7 +181,7 @@ def test_longest_chain_rule():
     log.info('[block 3\'] CURRENT BLOCK HEAD:{}'.format(encode_hex(t.chain.head_hash)))
     log.info('[block 3\'] CURRENT SHARD HEAD:{}'.format(encode_hex(t.chain.shards[shard_id].head_hash)))
     assert t.chain.shards[shard_id].get_score(t.chain.shards[shard_id].head) == 2
-    assert t.chain.get_score(t.chain.head) == 18
+    assert t.chain.get_score(t.chain.head) == 42
     assert t.chain.shards[shard_id].head_hash == collation_BC.hash
 
     # [block 3]: doesn't include collation
@@ -190,13 +191,13 @@ def test_longest_chain_rule():
     log.info('[block 3] CURRENT BLOCK HEAD:{}'.format(encode_hex(t.chain.head_hash)))
     log.info('[block 3] CURRENT SHARD HEAD:{}'.format(encode_hex(t.chain.shards[shard_id].head_hash)))
     assert t.chain.shards[shard_id].get_score(t.chain.shards[shard_id].head) == 2
-    assert t.chain.get_score(t.chain.head) == 18
+    assert t.chain.get_score(t.chain.head) == 42
     assert t.chain.shards[shard_id].head_hash == collation_BC.hash
 
     # [block 4]: doesn't include collation
     t.mine(5)
     log.info('[block 4] CURRENT BLOCK HEAD:{}'.format(encode_hex(t.chain.head_hash)))
     log.info('[block 4] CURRENT SHARD HEAD:{}'.format(encode_hex(t.chain.shards[shard_id].head_hash)))
     assert t.chain.shards[shard_id].get_score(t.chain.shards[shard_id].head) == 1
-    assert t.chain.get_score(t.chain.head) == 23
+    assert t.chain.get_score(t.chain.head) == 47
     assert t.chain.shards[shard_id].head_hash == collation_AB.hash

sharding/tests/test_shard_chain.py

@@ -27,7 +27,7 @@ def chain(shard_id, k0_deposit=True):
     if k0_deposit:
         # deposit
         c.sharding_deposit(privkey, valcode_addr)
-        c.mine(1)
+        c.mine(sharding_config['SHUFFLING_CYCLE_LENGTH'])
     c.add_test_shard(shard_id)
     return c