Merge branch 'develop' into feature/pubsub_message_ordering

# Conflicts: # docs/dockerhub.md
lKapKDcyTPe · Dec 19, 2021 · 8f93376 · 8f93376
2 parents ecc4484 + e0ca8f9
commit 8f93376
Show file tree

Hide file tree

Showing 43 changed files with 320 additions and 75 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Ethereum ETL
 
-[![Build Status](https://travis-ci.org/blockchain-etl/ethereum-etl.png)](https://travis-ci.org/blockchain-etl/ethereum-etl)
+[![Build Status](https://app.travis-ci.com/blockchain-etl/ethereum-etl.svg?branch=develop)](https://travis-ci.com/github/blockchain-etl/ethereum-etl)
 [![Join the chat at https://gitter.im/ethereum-eth](https://badges.gitter.im/ethereum-etl.svg)](https://gitter.im/ethereum-etl/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Telegram](https://img.shields.io/badge/telegram-join%20chat-blue.svg)](https://t.me/joinchat/GsMpbA3mv1OJ6YMp3T5ORQ)
 [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/wukrezR)
@@ -64,6 +64,7 @@ For the latest version, check out the repo and call
 - [Schema](https://ethereum-etl.readthedocs.io/en/latest/schema/)
 - [Command Reference](https://ethereum-etl.readthedocs.io/en/latest/commands/)
 - [Documentation](https://ethereum-etl.readthedocs.io/)
+- [Public Datasets in BigQuery](https://github.com/blockchain-etl/public-datasets)  
 - [Exporting the Blockchain](https://ethereum-etl.readthedocs.io/en/latest/exporting-the-blockchain/)
 - [Querying in Amazon Athena](https://ethereum-etl.readthedocs.io/en/latest/amazon-athena/)
 - [Querying in Google BigQuery](https://ethereum-etl.readthedocs.io/en/latest/google-bigquery/)
@@ -77,8 +78,9 @@ For the latest version, check out the repo and call
 ```bash
 > pip3 install -e .[dev,streaming]
 > export ETHEREUM_ETL_RUN_SLOW_TESTS=True
+> export PROVIDER_URL=<your_porvider_uri>
 > pytest -vv
-```
+``` 
 
 ### Running Tox Tests
 
@@ -89,7 +91,7 @@ For the latest version, check out the repo and call
 
 ## Running in Docker
 
-1. Install Docker https://docs.docker.com/install/
+1. Install Docker: https://docs.docker.com/install/
 
 2. Build a docker image
 
@@ -111,4 +113,4 @@ For the latest version, check out the repo and call
 
 ## Projects using Ethereum ETL
 * [Google](https://goo.gl/oY5BCQ) - Public BigQuery Ethereum datasets
-* [Nansen by D5](https://nansen.d5.ai/?ref=ethereumetl) - Analytics platform for Ethereum
+* [Nansen](https://www.nansen.ai/?ref=ethereumetl) - Analytics platform for Ethereum
diff --git a/blockchainetl/jobs/exporters/composite_item_exporter.py b/blockchainetl/jobs/exporters/composite_item_exporter.py
@@ -24,17 +24,20 @@
 from blockchainetl.atomic_counter import AtomicCounter
 from blockchainetl.exporters import CsvItemExporter, JsonLinesItemExporter
 from blockchainetl.file_utils import get_file_handle, close_silently
+from blockchainetl.jobs.exporters.converters.composite_item_converter import CompositeItemConverter
 
 
 class CompositeItemExporter:
-    def __init__(self, filename_mapping, field_mapping=None):
+    def __init__(self, filename_mapping, field_mapping=None, converters=()):
         self.filename_mapping = filename_mapping
         self.field_mapping = field_mapping or {}
 
         self.file_mapping = {}
         self.exporter_mapping = {}
         self.counter_mapping = {}
 
+        self.converter = CompositeItemConverter(converters)
+
         self.logger = logging.getLogger('CompositeItemExporter')
 
     def open(self):
@@ -62,7 +65,7 @@ def export_item(self, item):
         exporter = self.exporter_mapping.get(item_type)
         if exporter is None:
             raise ValueError('Exporter for item type {} not found'.format(item_type))
-        exporter.export_item(item)
+        exporter.export_item(self.converter.convert_item(item))
 
         counter = self.counter_mapping.get(item_type)
         if counter is not None:

diff --git a/blockchainetl/jobs/exporters/converters/__init__.py b/blockchainetl/jobs/exporters/converters/__init__.py
diff --git a/blockchainetl/jobs/exporters/converters/composite_item_converter.py b/blockchainetl/jobs/exporters/converters/composite_item_converter.py
@@ -37,6 +37,9 @@ def __init__(self, converters=()):
         self.converters = converters
 
     def convert_item(self, item):
+        if self.converters is None:
+            return item
+
         for converter in self.converters:
             item = converter.convert_item(item)
         return item
diff --git a/blockchainetl/jobs/exporters/converters/int_to_string_item_converter.py b/blockchainetl/jobs/exporters/converters/int_to_string_item_converter.py
@@ -0,0 +1,46 @@
+#  MIT License
+#
+#  Copyright (c) 2018 Evgeny Medvedev, [email protected]
+#
+#  Permission is hereby granted, free of charge, to any person obtaining a copy
+#  of this software and associated documentation files (the "Software"), to deal
+#  in the Software without restriction, including without limitation the rights
+#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  copies of the Software, and to permit persons to whom the Software is
+#  furnished to do so, subject to the following conditions:
+#
+#  The above copyright notice and this permission notice shall be included in all
+#  copies or substantial portions of the Software.
+#
+#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  SOFTWARE.
+
+# MIT License
+#
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+
+from blockchainetl.jobs.exporters.converters.simple_item_converter import SimpleItemConverter
+
+
+class IntToStringItemConverter(SimpleItemConverter):
+
+    def __init__(self, keys=None):
+        self.keys = set(keys) if keys else None
+
+    def convert_field(self, key, value):
+        if isinstance(value, int) and (self.keys is None or key in self.keys):
+            return str(value)
+        else:
+            return value
diff --git a/blockchainetl/jobs/exporters/converters/simple_item_converter.py b/blockchainetl/jobs/exporters/converters/simple_item_converter.py
@@ -33,9 +33,6 @@
 
 class SimpleItemConverter:
 
-    def __init__(self, converters=()):
-        self.converters = converters
-
     def convert_item(self, item):
         return {
             key: self.convert_field(key, value) for key, value in item.items()

diff --git a/blockchainetl/logging_utils.py b/blockchainetl/logging_utils.py
@@ -7,3 +7,5 @@ def logging_basic_config(filename=None):
         logging.basicConfig(level=logging.INFO, format=format, filename=filename)
     else:
         logging.basicConfig(level=logging.INFO, format=format)
+
+    logging.getLogger('ethereum_dasm.evmdasm').setLevel(logging.ERROR)
diff --git a/docs/dockerhub.md b/docs/dockerhub.md
@@ -1,7 +1,7 @@
 # Uploading to Docker Hub
 
 ```bash
-ETHEREUMETL_VERSION=1.6.0-ordering2
+ETHEREUMETL_VERSION=1.7.4
 docker build -t ethereum-etl:${ETHEREUMETL_VERSION} -f Dockerfile .
 docker tag ethereum-etl:${ETHEREUMETL_VERSION} blockchainetl/ethereum-etl:${ETHEREUMETL_VERSION}
 docker push blockchainetl/ethereum-etl:${ETHEREUMETL_VERSION}

diff --git a/docs/exporting-the-blockchain.md b/docs/exporting-the-blockchain.md
@@ -14,7 +14,8 @@ export the data ~40 times faster, you will need to set up a local Ethereum node:
 Make sure it downloaded the blocks that you need by executing `eth.syncing` in the JS console.
 You can export blocks below `currentBlock`,
 there is no need to wait until the full sync as the state is not needed (unless you also need contracts bytecode
-and token details; for those you need to wait until the full sync).
+and token details; for those you need to wait until the full sync). Note that you may need to wait for another day or 
+   two for the node to download the states. See this issue https://github.com/blockchain-etl/ethereum-etl/issues/265#issuecomment-970451522
 
 1. Install Ethereum ETL: `> pip3 install ethereum-etl`
 

diff --git a/docs/schema.md b/docs/schema.md
@@ -22,6 +22,7 @@ gas_limit         | bigint             |
 gas_used          | bigint             |
 timestamp         | bigint             |
 transaction_count | bigint             |
+base_fee_per_gas     | bigint             |
 
 ---
 
@@ -41,6 +42,9 @@ gas              | bigint      |
 gas_price        | bigint      |
 input            | hex_string  |
 block_timestamp  | bigint      |
+max_fee_per_gas  | bigint      |
+max_priority_fee_per_gas | bigint |
+transaction_type | bigint |
 
 ---
 
@@ -71,6 +75,7 @@ gas_used                     | bigint      |
 contract_address             | address     |
 root                         | hex_string  |
 status                       | bigint      |
+effective_gas_price          | bigint      |
 
 ---
 

diff --git a/ethereumetl/cli/__init__.py b/ethereumetl/cli/__init__.py
@@ -19,6 +19,10 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+
+from blockchainetl.logging_utils import logging_basic_config
+logging_basic_config()
+
 import click
 
 from ethereumetl.cli.export_all import export_all
@@ -44,7 +48,7 @@
 
 
 @click.group()
-@click.version_option(version='1.6.0')
+@click.version_option(version='1.7.4')
 @click.pass_context
 def cli(ctx):
     pass

diff --git a/ethereumetl/cli/export_contracts.py b/ethereumetl/cli/export_contracts.py
@@ -36,7 +36,7 @@
 
 @click.command(context_settings=dict(help_option_names=['-h', '--help']))
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to filter at a time.')
-@click.option('-c', '--contract-addresses', required=True, type=str,
+@click.option('-ca', '--contract-addresses', required=True, type=str,
               help='The file containing contract addresses, one per line.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')

diff --git a/ethereumetl/cli/extract_token_transfers.py b/ethereumetl/cli/extract_token_transfers.py
@@ -26,6 +26,7 @@
 import json
 
 from blockchainetl.file_utils import smart_open
+from blockchainetl.jobs.exporters.converters.int_to_string_item_converter import IntToStringItemConverter
 from ethereumetl.jobs.exporters.token_transfers_item_exporter import token_transfers_item_exporter
 from ethereumetl.jobs.extract_token_transfers_job import ExtractTokenTransfersJob
 from blockchainetl.logging_utils import logging_basic_config
@@ -38,17 +39,19 @@
 @click.option('-b', '--batch-size', default=100, show_default=True, type=int, help='The number of blocks to filter at a time.')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
-def extract_token_transfers(logs, batch_size, output, max_workers):
+@click.option('--values-as-strings', default=False, show_default=True, is_flag=True, help='Whether to convert values to strings.')
+def extract_token_transfers(logs, batch_size, output, max_workers, values_as_strings=False):
     """Extracts ERC20/ERC721 transfers from logs file."""
     with smart_open(logs, 'r') as logs_file:
         if logs.endswith('.json'):
             logs_reader = (json.loads(line) for line in logs_file)
         else:
             logs_reader = csv.DictReader(logs_file)
+        converters = [IntToStringItemConverter(keys=['value'])] if values_as_strings else []
         job = ExtractTokenTransfersJob(
             logs_iterable=logs_reader,
             batch_size=batch_size,
             max_workers=max_workers,
-            item_exporter=token_transfers_item_exporter(output))
+            item_exporter=token_transfers_item_exporter(output, converters=converters))
 
         job.run()
diff --git a/ethereumetl/cli/extract_tokens.py b/ethereumetl/cli/extract_tokens.py
@@ -27,6 +27,7 @@
 import click
 from blockchainetl.csv_utils import set_max_field_size_limit
 from blockchainetl.file_utils import smart_open
+from blockchainetl.jobs.exporters.converters.int_to_string_item_converter import IntToStringItemConverter
 from ethereumetl.jobs.exporters.tokens_item_exporter import tokens_item_exporter
 from ethereumetl.jobs.extract_tokens_job import ExtractTokensJob
 from blockchainetl.logging_utils import logging_basic_config
@@ -44,7 +45,8 @@
                    'file://$HOME/Library/Ethereum/geth.ipc or https://mainnet.infura.io')
 @click.option('-o', '--output', default='-', show_default=True, type=str, help='The output file. If not specified stdout is used.')
 @click.option('-w', '--max-workers', default=5, show_default=True, type=int, help='The maximum number of workers.')
-def extract_tokens(contracts, provider_uri, output, max_workers):
+@click.option('--values-as-strings', default=False, show_default=True, is_flag=True, help='Whether to convert values to strings.')
+def extract_tokens(contracts, provider_uri, output, max_workers, values_as_strings=False):
     """Extracts tokens from contracts file."""
 
     set_max_field_size_limit()
@@ -54,10 +56,11 @@ def extract_tokens(contracts, provider_uri, output, max_workers):
             contracts_iterable = (json.loads(line) for line in contracts_file)
         else:
             contracts_iterable = csv.DictReader(contracts_file)
+        converters = [IntToStringItemConverter(keys=['decimals', 'total_supply'])] if values_as_strings else []
         job = ExtractTokensJob(
             contracts_iterable=contracts_iterable,
             web3=ThreadLocalProxy(lambda: Web3(get_provider_from_uri(provider_uri))),
             max_workers=max_workers,
-            item_exporter=tokens_item_exporter(output))
+            item_exporter=tokens_item_exporter(output, converters))
 
         job.run()
diff --git a/ethereumetl/domain/block.py b/ethereumetl/domain/block.py
@@ -43,3 +43,4 @@ def __init__(self):
 
         self.transactions = []
         self.transaction_count = 0
+        self.base_fee_per_gas = 0
diff --git a/ethereumetl/domain/receipt.py b/ethereumetl/domain/receipt.py
@@ -33,3 +33,4 @@ def __init__(self):
         self.logs = []
         self.root = None
         self.status = None
+        self.effective_gas_price = None
diff --git a/ethereumetl/domain/transaction.py b/ethereumetl/domain/transaction.py
@@ -34,3 +34,6 @@ def __init__(self):
         self.gas = None
         self.gas_price = None
         self.input = None
+        self.max_fee_per_gas = None
+        self.max_priority_fee_per_gas = None
+        self.transaction_type = None
Original file line number	Diff line number	Diff line change
Expand Up		@@ -43,3 +43,4 @@ def __init__(self):

		self.transactions = []
		self.transaction_count = 0
		self.base_fee_per_gas = 0