Merge pull request blockchain-etl#263 from blockchain-etl/eip1559-fields

EIP-1559 fields

blockchainetl/jobs/exporters/composite_item_exporter.py

@@ -24,17 +24,20 @@
 from blockchainetl.atomic_counter import AtomicCounter
 from blockchainetl.exporters import CsvItemExporter, JsonLinesItemExporter
 from blockchainetl.file_utils import get_file_handle, close_silently
+from blockchainetl.jobs.exporters.converters.composite_item_converter import CompositeItemConverter
 
 
 class CompositeItemExporter:
-    def __init__(self, filename_mapping, field_mapping=None):
+    def __init__(self, filename_mapping, field_mapping=None, converters=()):
         self.filename_mapping = filename_mapping
         self.field_mapping = field_mapping or {}
 
         self.file_mapping = {}
         self.exporter_mapping = {}
         self.counter_mapping = {}
 
+        self.converter = CompositeItemConverter(converters)
+
         self.logger = logging.getLogger('CompositeItemExporter')
 
     def open(self):
@@ -62,7 +65,7 @@ def export_item(self, item):
         exporter = self.exporter_mapping.get(item_type)
         if exporter is None:
             raise ValueError('Exporter for item type {} not found'.format(item_type))
-        exporter.export_item(item)
+        exporter.export_item(self.converter.convert_item(item))
 
         counter = self.counter_mapping.get(item_type)
         if counter is not None:

blockchainetl/jobs/exporters/converters/composite_item_converter.py

@@ -37,6 +37,9 @@ def __init__(self, converters=()):
         self.converters = converters
 
     def convert_item(self, item):
+        if self.converters is None:
+            return item
+
         for converter in self.converters:
             item = converter.convert_item(item)
         return item

blockchainetl/jobs/exporters/converters/int_to_string_item_converter.py

@@ -0,0 +1,46 @@
+#  MIT License
+#
+#  Copyright (c) 2018 Evgeny Medvedev, [email protected]
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  SOFTWARE.
+
+# MIT License
+#
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+
+from blockchainetl.jobs.exporters.converters.simple_item_converter import SimpleItemConverter
+
+
+class IntToStringItemConverter(SimpleItemConverter):
+
+    def __init__(self, keys=None):
+        self.keys = set(keys) if keys else None
+
+    def convert_field(self, key, value):
+        if isinstance(value, int) and (self.keys is None or key in self.keys):
+            return str(value)
+        else:
+            return value

blockchainetl/jobs/exporters/converters/simple_item_converter.py

@@ -33,9 +33,6 @@
 
 class SimpleItemConverter:
 
-    def __init__(self, converters=()):
-        self.converters = converters
-
     def convert_item(self, item):
         return {
             key: self.convert_field(key, value) for key, value in item.items()

docs/schema.md

@@ -22,6 +22,7 @@ gas_limit         | bigint             |
 gas_used          | bigint             |
 timestamp         | bigint             |
 transaction_count | bigint             |
+base_fee_per_gas     | bigint             |
 
 ---
 
@@ -41,6 +42,9 @@ gas              | bigint      |
 gas_price        | bigint      |
 input            | hex_string  |
 block_timestamp  | bigint      |
+max_fee_per_gas  | bigint      |
+max_priority_fee_per_gas | bigint |
+transaction_type | bigint |
 
 ---
 
@@ -71,6 +75,7 @@ gas_used                     | bigint      |
 contract_address             | address     |
 root                         | hex_string  |
 status                       | bigint      |
+effective_gas_price          | bigint      |
 
 ---
 

ethereumetl/cli/extract_token_transfers.py

@@ -26,6 +26,7 @@
 import json
 
 from blockchainetl.file_utils import smart_open
+from blockchainetl.jobs.exporters.converters.int_to_string_item_converter import IntToStringItemConverter
 from ethereumetl.jobs.exporters.token_transfers_item_exporter import token_transfers_item_exporter
 from ethereumetl.jobs.extract_token_transfers_job import ExtractTokenTransfersJob
 from blockchainetl.logging_utils import logging_basic_config
@@ -38,17 +39,19 @@
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to filter at a time.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
-def extract_token_transfers(logs, batch_size, output, max_workers):
+@click.option('--values-as-strings', default=False, show_default=True, is_flag=True, help='Whether to convert values to strings.')
+def extract_token_transfers(logs, batch_size, output, max_workers, values_as_strings=False):
     """Extracts ERC20/ERC721 transfers from logs file."""
     with smart_open(logs, 'r') as logs_file:
         if logs.endswith('.json'):
             logs_reader = (json.loads(line) for line in logs_file)
         else:
             logs_reader = csv.DictReader(logs_file)
+        converters = [IntToStringItemConverter(keys=['value'])] if values_as_strings else []
         job = ExtractTokenTransfersJob(
             logs_iterable=logs_reader,
             batch_size=batch_size,
             max_workers=max_workers,
-            item_exporter=token_transfers_item_exporter(output))
+            item_exporter=token_transfers_item_exporter(output, converters=converters))
 
         job.run()

ethereumetl/cli/extract_tokens.py

@@ -27,6 +27,7 @@
 import click
 from blockchainetl.csv_utils import set_max_field_size_limit
 from blockchainetl.file_utils import smart_open
+from blockchainetl.jobs.exporters.converters.int_to_string_item_converter import IntToStringItemConverter
 from ethereumetl.jobs.exporters.tokens_item_exporter import tokens_item_exporter
 from ethereumetl.jobs.extract_tokens_job import ExtractTokensJob
 from blockchainetl.logging_utils import logging_basic_config
@@ -44,7 +45,8 @@
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
-def extract_tokens(contracts, provider_uri, output, max_workers):
+@click.option('--values-as-strings', default=False, show_default=True, is_flag=True, help='Whether to convert values to strings.')
+def extract_tokens(contracts, provider_uri, output, max_workers, values_as_strings=False):
     """Extracts tokens from contracts file."""
 
     set_max_field_size_limit()
@@ -54,10 +56,11 @@ def extract_tokens(contracts, provider_uri, output, max_workers):
             contracts_iterable = (json.loads(line) for line in contracts_file)
         else:
             contracts_iterable = csv.DictReader(contracts_file)
+        converters = [IntToStringItemConverter(keys=['decimals', 'total_supply'])] if values_as_strings else []
         job = ExtractTokensJob(
             contracts_iterable=contracts_iterable,
             web3=ThreadLocalProxy(lambda: Web3(get_provider_from_uri(provider_uri))),
             max_workers=max_workers,
-            item_exporter=tokens_item_exporter(output))
+            item_exporter=tokens_item_exporter(output, converters))
 
         job.run()

ethereumetl/domain/block.py

@@ -43,3 +43,4 @@ def __init__(self):
 
         self.transactions = []
         self.transaction_count = 0
+        self.base_fee_per_gas = 0

ethereumetl/domain/receipt.py

@@ -33,3 +33,4 @@ def __init__(self):
         self.logs = []
         self.root = None
         self.status = None
+        self.effective_gas_price = None

ethereumetl/domain/transaction.py

@@ -34,3 +34,6 @@ def __init__(self):
         self.gas = None
         self.gas_price = None
         self.input = None
+        self.max_fee_per_gas = None
+        self.max_priority_fee_per_gas = None
+        self.transaction_type = None

ethereumetl/jobs/exporters/blocks_and_transactions_item_exporter.py

@@ -41,7 +41,8 @@
     'gas_limit',
     'gas_used',
     'timestamp',
-    'transaction_count'
+    'transaction_count',
+    'base_fee_per_gas'
 ]
 
 TRANSACTION_FIELDS_TO_EXPORT = [
@@ -56,7 +57,10 @@
     'gas',
     'gas_price',
     'input',
-    'block_timestamp'
+    'block_timestamp',
+    'max_fee_per_gas',
+    'max_priority_fee_per_gas',
+    'transaction_type'
 ]
 
 

ethereumetl/jobs/exporters/receipts_and_logs_item_exporter.py

@@ -32,7 +32,8 @@
     'gas_used',
     'contract_address',
     'root',
-    'status'
+    'status',
+    'effective_gas_price'
 ]
 
 LOG_FIELDS_TO_EXPORT = [

ethereumetl/jobs/exporters/token_transfers_item_exporter.py

@@ -34,12 +34,13 @@
 ]
 
 
-def token_transfers_item_exporter(token_transfer_output):
+def token_transfers_item_exporter(token_transfer_output, converters=()):
     return CompositeItemExporter(
         filename_mapping={
             'token_transfer': token_transfer_output
         },
         field_mapping={
             'token_transfer': FIELDS_TO_EXPORT
-        }
+        },
+        converters=converters
     )

ethereumetl/jobs/exporters/tokens_item_exporter.py

@@ -33,12 +33,13 @@
 ]
 
 
-def tokens_item_exporter(tokens_output):
+def tokens_item_exporter(tokens_output, converters=()):
     return CompositeItemExporter(
         filename_mapping={
             'token': tokens_output
         },
         field_mapping={
             'token': FIELDS_TO_EXPORT
-        }
+        },
+        converters=converters
     )

ethereumetl/mappers/block_mapper.py

@@ -52,6 +52,7 @@ def json_dict_to_block(self, json_dict):
         block.gas_limit = hex_to_dec(json_dict.get('gasLimit'))
         block.gas_used = hex_to_dec(json_dict.get('gasUsed'))
         block.timestamp = hex_to_dec(json_dict.get('timestamp'))
+        block.base_fee_per_gas = hex_to_dec(json_dict.get('baseFeePerGas'))
 
         if 'transactions' in json_dict:
             block.transactions = [
@@ -85,4 +86,5 @@ def block_to_dict(self, block):
             'gas_used': block.gas_used,
             'timestamp': block.timestamp,
             'transaction_count': block.transaction_count,
+            'base_fee_per_gas': block.base_fee_per_gas
         }

ethereumetl/mappers/receipt_mapper.py

@@ -48,6 +48,8 @@ def json_dict_to_receipt(self, json_dict):
         receipt.root = json_dict.get('root')
         receipt.status = hex_to_dec(json_dict.get('status'))
 
+        receipt.effective_gas_price = hex_to_dec(json_dict.get('effectiveGasPrice'))
+
         if 'logs' in json_dict:
             receipt.logs = [
                 self.receipt_log_mapper.json_dict_to_receipt_log(log) for log in json_dict['logs']
@@ -66,5 +68,6 @@ def receipt_to_dict(self, receipt):
             'gas_used': receipt.gas_used,
             'contract_address': receipt.contract_address,
             'root': receipt.root,
-            'status': receipt.status
+            'status': receipt.status,
+            'effective_gas_price': receipt.effective_gas_price
         }

ethereumetl/mappers/transaction_mapper.py

@@ -40,6 +40,9 @@ def json_dict_to_transaction(self, json_dict, **kwargs):
         transaction.gas = hex_to_dec(json_dict.get('gas'))
         transaction.gas_price = hex_to_dec(json_dict.get('gasPrice'))
         transaction.input = json_dict.get('input')
+        transaction.max_fee_per_gas = hex_to_dec(json_dict.get('maxFeePerGas'))
+        transaction.max_priority_fee_per_gas = hex_to_dec(json_dict.get('maxPriorityFeePerGas'))
+        transaction.transaction_type = hex_to_dec(json_dict.get('type'))
         return transaction
 
     def transaction_to_dict(self, transaction):
@@ -57,4 +60,7 @@ def transaction_to_dict(self, transaction):
             'gas': transaction.gas,
             'gas_price': transaction.gas_price,
             'input': transaction.input,
+            'max_fee_per_gas': transaction.max_fee_per_gas,
+            'max_priority_fee_per_gas': transaction.max_priority_fee_per_gas,
+            'transaction_type': transaction.transaction_type
         }

ethereumetl/service/token_transfer_extractor.py

@@ -37,8 +37,7 @@ def extract_transfer_from_log(self, receipt_log):
 
         topics = receipt_log.topics
         if topics is None or len(topics) < 1:
-            logger.warning("Topics are empty in log {} of transaction {}".format(receipt_log.log_index,
-                                                                                 receipt_log.transaction_hash))
+            # This is normal, topics can be empty for anonymous events
             return None
 
         if topics[0] == TRANSFER_EVENT_TOPIC:

ethereumetl/streaming/enrich.py

@@ -73,14 +73,18 @@ def enrich_transactions(transactions, receipts):
             'input',
             'block_timestamp',
             'block_number',
-            'block_hash'
+            'block_hash',
+            'max_fee_per_gas',
+            'max_priority_fee_per_gas',
+            'transaction_type'
         ],
         right_fields=[
             ('cumulative_gas_used', 'receipt_cumulative_gas_used'),
             ('gas_used', 'receipt_gas_used'),
             ('contract_address', 'receipt_contract_address'),
             ('root', 'receipt_root'),
-            ('status', 'receipt_status')
+            ('status', 'receipt_status'),
+            ('effective_gas_price', 'effective_gas_price')
         ]))
 
     if len(result) != len(transactions):

setup.py

@@ -37,7 +37,7 @@ def read(fname):
         'eth-utils==1.10.0',
         'eth-abi==1.3.0',
         # TODO: This has to be removed when "ModuleNotFoundError: No module named 'eth_utils.toolz'" is fixed at eth-abi
-        'python-dateutil==2.7.0',
+        'python-dateutil==2.8.0',
         'click==7.1.2',
         'ethereum-dasm==0.1.4',
         'base58',

tests/resources/test_export_blocks_job/block_with_logs/expected_blocks.csv

@@ -1,2 +1,2 @@
-number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,receipts_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count
-483920,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,0x2610dc6eb941f4bcbddfd2362b999087ccd956e978f0ece4f8da96851283a2ba,0x57a633e01197dc86,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000800000000000000000000000000000800000000000000000000000000000008000000000000000000000000000000000000021000000080000000004000008000000000000000000000400000000000000000000000000000000400000000000000000000000000000000000000010000000000000000000000000000000000000000400000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000010000000000000000000000000000000000000000000000004000000000000000000000000000000000000040080000,0x2744d46ab0647ed91a9bbd08e19d3bb67491067e8cbe04a276ad2afde5ecd65e,0x48b17dd0031aa97d748a886c912539de22997e861d631fd1eb6509fbabef9651,0xada95dd1e1590fe095e67c58f41d633193b238e0e0c588de46682db595738f0b,0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5,7298514125186,2571481026230204460,1113,0xd783010203844765746887676f312e342e32856c696e7578,3141592,143706,1446561880,4
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,receipts_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count,base_fee_per_gas
+483920,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,0x2610dc6eb941f4bcbddfd2362b999087ccd956e978f0ece4f8da96851283a2ba,0x57a633e01197dc86,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000800000000000000000000000000000800000000000000000000000000000008000000000000000000000000000000000000021000000080000000004000008000000000000000000000400000000000000000000000000000000400000000000000000000000000000000000000010000000000000000000000000000000000000000400000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000010000000000000000000000000000000000000000000000004000000000000000000000000000000000000040080000,0x2744d46ab0647ed91a9bbd08e19d3bb67491067e8cbe04a276ad2afde5ecd65e,0x48b17dd0031aa97d748a886c912539de22997e861d631fd1eb6509fbabef9651,0xada95dd1e1590fe095e67c58f41d633193b238e0e0c588de46682db595738f0b,0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5,7298514125186,2571481026230204460,1113,0xd783010203844765746887676f312e342e32856c696e7578,3141592,143706,1446561880,4,

tests/resources/test_export_blocks_job/block_with_logs/expected_transactions.csv

@@ -1,5 +1,5 @@
-hash,nonce,block_hash,block_number,transaction_index,from_address,to_address,value,gas,gas_price,input,block_timestamp
-0x04cbcb236043d8fb7839e07bbc7f5eed692fb2ca55d897f1101eac3e3ad4fab8,12,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,0,0x1b63142628311395ceafeea5667e7c9026c862ca,0xf4eced2f682ce333f96f2d8966c613ded8fc95dd,0,150853,50000000000,0xa9059cbb000000000000000000000000ac4df82fe37ea2187bc8c011a23d743b4f39019a00000000000000000000000000000000000000000000000000000000000186a0,1446561880
-0xcea6f89720cc1d2f46cc7a935463ae0b99dd5fad9c91bb7357de5421511cee49,84,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,1,0x9b22a80d5c7b3374a05b446081f97d0a34079e7f,0xf4eced2f682ce333f96f2d8966c613ded8fc95dd,0,150853,50000000000,0xa9059cbb00000000000000000000000066f183060253cfbe45beff1e6e7ebbe318c81e560000000000000000000000000000000000000000000000000000000000030d40,1446561880
-0x463d53f0ad57677a3b430a007c1c31d15d62c37fab5eee598551697c297c235c,88,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,2,0x9df428a91ff0f3635c8f0ce752933b9788926804,0x9e669f970ec0f49bb735f20799a7e7c4a1c274e2,11000440000000000,90000,50000000000,0x,1446561880
-0x05287a561f218418892ab053adfb3d919860988b19458c570c5c30f51c146f02,20085,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,3,0x2a65aca4d5fc5b5c859090a6c34d164135398226,0x743b8aeedc163c0e3a0fe9f3910d146c48e70da8,1530219620000000000,90000,50000000000,0x,1446561880
+hash,nonce,block_hash,block_number,transaction_index,from_address,to_address,value,gas,gas_price,input,block_timestamp,max_fee_per_gas,max_priority_fee_per_gas,transaction_type
+0x04cbcb236043d8fb7839e07bbc7f5eed692fb2ca55d897f1101eac3e3ad4fab8,12,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,0,0x1b63142628311395ceafeea5667e7c9026c862ca,0xf4eced2f682ce333f96f2d8966c613ded8fc95dd,0,150853,50000000000,0xa9059cbb000000000000000000000000ac4df82fe37ea2187bc8c011a23d743b4f39019a00000000000000000000000000000000000000000000000000000000000186a0,1446561880,,,
+0xcea6f89720cc1d2f46cc7a935463ae0b99dd5fad9c91bb7357de5421511cee49,84,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,1,0x9b22a80d5c7b3374a05b446081f97d0a34079e7f,0xf4eced2f682ce333f96f2d8966c613ded8fc95dd,0,150853,50000000000,0xa9059cbb00000000000000000000000066f183060253cfbe45beff1e6e7ebbe318c81e560000000000000000000000000000000000000000000000000000000000030d40,1446561880,,,
+0x463d53f0ad57677a3b430a007c1c31d15d62c37fab5eee598551697c297c235c,88,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,2,0x9df428a91ff0f3635c8f0ce752933b9788926804,0x9e669f970ec0f49bb735f20799a7e7c4a1c274e2,11000440000000000,90000,50000000000,0x,1446561880,,,
+0x05287a561f218418892ab053adfb3d919860988b19458c570c5c30f51c146f02,20085,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,483920,3,0x2a65aca4d5fc5b5c859090a6c34d164135398226,0x743b8aeedc163c0e3a0fe9f3910d146c48e70da8,1530219620000000000,90000,50000000000,0x,1446561880,,,