From dc2c3ff1b39ca77777bd142a47656810bc71f3fd Mon Sep 17 00:00:00 2001
From: Evgeny Medvedev <medvedev1088@gmail.com>
Date: Sat, 4 Aug 2018 23:10:01 +0700
Subject: [PATCH] Remove block_ prefix from CSV and tables
---
README.md | 36 +++++++++----------
.../blocks_and_transactions_item_exporter.py | 34 +++++++++---------
ethereumetl/mappers/block_mapper.py | 34 +++++++++---------
schemas/aws/blocks.sql | 34 +++++++++---------
schemas/aws/parquet/parquet_blocks.sql | 34 +++++++++---------
schemas/gcp/blocks.json | 34 +++++++++---------
schemas/gcp/transactions_join_receipts.sql | 10 +++---
.../block_with_logs/expected_blocks.csv | 2 +-
.../expected_blocks.csv | 2 +-
.../expected_blocks.csv | 2 +-
10 files changed, 111 insertions(+), 111 deletions(-)
diff --git a/README.md b/README.md
index b6fa3f841..e4b99a40f 100644
--- a/README.md
+++ b/README.md
@@ -50,23 +50,23 @@ Read this article https://medium.com/@medvedev1088/exporting-and-analyzing-ether
Column | Type |
------------------------|--------------------|
-block_number | bigint |
-block_hash | hex_string |
-block_parent_hash | hex_string |
-block_nonce | hex_string |
-block_sha3_uncles | hex_string |
-block_logs_bloom | hex_string |
-block_transactions_root | hex_string |
-block_state_root | hex_string |
-block_miner | address |
-block_difficulty | numeric |
-block_total_difficulty | numeric |
-block_size | bigint |
-block_extra_data | hex_string |
-block_gas_limit | bigint |
-block_gas_used | bigint |
-block_timestamp | bigint |
-block_transaction_count | bigint |
+number | bigint |
+hash | hex_string |
+parent_hash | hex_string |
+nonce | hex_string |
+sha3_uncles | hex_string |
+logs_bloom | hex_string |
+transactions_root | hex_string |
+state_root | hex_string |
+miner | address |
+difficulty | numeric |
+total_difficulty | numeric |
+size | bigint |
+extra_data | hex_string |
+gas_limit | bigint |
+gas_used | bigint |
+timestamp | bigint |
+transaction_count | bigint |
### transactions.csv
@@ -445,7 +445,7 @@ Note that NEWLINE_DELIMITED_JSON is used to support REPEATED mode for the column
Join `transactions` and `receipts`:
```bash
-> bq mk --table --description "Exported using https://github.com/medvedev1088/ethereum-etl" --time_partitioning_field block_timestamp_partition ethereum.transactions_join_receipts ./schemas/gcp/transactions_join_receipts.json
+> bq mk --table --description "Exported using https://github.com/medvedev1088/ethereum-etl" --time_partitioning_field timestamp_partition ethereum.transactions_join_receipts ./schemas/gcp/transactions_join_receipts.json
> bq --location=US query --replace --destination_table ethereum.transactions_join_receipts --use_legacy_sql=false "$(cat ./schemas/gcp/transactions_join_receipts.sql | tr '\n' ' ')"
```
diff --git a/ethereumetl/jobs/exporters/blocks_and_transactions_item_exporter.py b/ethereumetl/jobs/exporters/blocks_and_transactions_item_exporter.py
index f2b03f309..c76fa1d36 100644
--- a/ethereumetl/jobs/exporters/blocks_and_transactions_item_exporter.py
+++ b/ethereumetl/jobs/exporters/blocks_and_transactions_item_exporter.py
@@ -24,23 +24,23 @@
from ethereumetl.jobs.exporters.composite_item_exporter import CompositeItemExporter
BLOCK_FIELDS_TO_EXPORT = [
- 'block_number',
- 'block_hash',
- 'block_parent_hash',
- 'block_nonce',
- 'block_sha3_uncles',
- 'block_logs_bloom',
- 'block_transactions_root',
- 'block_state_root',
- 'block_miner',
- 'block_difficulty',
- 'block_total_difficulty',
- 'block_size',
- 'block_extra_data',
- 'block_gas_limit',
- 'block_gas_used',
- 'block_timestamp',
- 'block_transaction_count'
+ 'number',
+ 'hash',
+ 'parent_hash',
+ 'nonce',
+ 'sha3_uncles',
+ 'logs_bloom',
+ 'transactions_root',
+ 'state_root',
+ 'miner',
+ 'difficulty',
+ 'total_difficulty',
+ 'size',
+ 'extra_data',
+ 'gas_limit',
+ 'gas_used',
+ 'timestamp',
+ 'transaction_count'
]
TRANSACTION_FIELDS_TO_EXPORT = [
diff --git a/ethereumetl/mappers/block_mapper.py b/ethereumetl/mappers/block_mapper.py
index 6c1b3bb9c..4a7e26ead 100644
--- a/ethereumetl/mappers/block_mapper.py
+++ b/ethereumetl/mappers/block_mapper.py
@@ -65,21 +65,21 @@ def json_dict_to_block(self, json_dict):
def block_to_dict(self, block):
return {
'type': 'block',
- 'block_number': block.number,
- 'block_hash': block.hash,
- 'block_parent_hash': block.parent_hash,
- 'block_nonce': block.nonce,
- 'block_sha3_uncles': block.sha3_uncles,
- 'block_logs_bloom': block.logs_bloom,
- 'block_transactions_root': block.transactions_root,
- 'block_state_root': block.state_root,
- 'block_miner': block.miner,
- 'block_difficulty': block.difficulty,
- 'block_total_difficulty': block.total_difficulty,
- 'block_size': block.size,
- 'block_extra_data': block.extra_data,
- 'block_gas_limit': block.gas_limit,
- 'block_gas_used': block.gas_used,
- 'block_timestamp': block.timestamp,
- 'block_transaction_count': block.transaction_count,
+ 'number': block.number,
+ 'hash': block.hash,
+ 'parent_hash': block.parent_hash,
+ 'nonce': block.nonce,
+ 'sha3_uncles': block.sha3_uncles,
+ 'logs_bloom': block.logs_bloom,
+ 'transactions_root': block.transactions_root,
+ 'state_root': block.state_root,
+ 'miner': block.miner,
+ 'difficulty': block.difficulty,
+ 'total_difficulty': block.total_difficulty,
+ 'size': block.size,
+ 'extra_data': block.extra_data,
+ 'gas_limit': block.gas_limit,
+ 'gas_used': block.gas_used,
+ 'timestamp': block.timestamp,
+ 'transaction_count': block.transaction_count,
}
diff --git a/schemas/aws/blocks.sql b/schemas/aws/blocks.sql
index 556424d09..46095e1ff 100644
--- a/schemas/aws/blocks.sql
+++ b/schemas/aws/blocks.sql
@@ -1,21 +1,21 @@
CREATE EXTERNAL TABLE IF NOT EXISTS blocks (
- block_number BIGINT,
- block_hash STRING,
- block_parent_hash STRING,
- block_nonce STRING,
- block_sha3_uncles STRING,
- block_logs_bloom STRING,
- block_transactions_root STRING,
- block_state_root STRING,
- block_miner STRING,
- block_difficulty DECIMAL(38,0),
- block_total_difficulty DECIMAL(38,0),
- block_size BIGINT,
- block_extra_data STRING,
- block_gas_limit BIGINT,
- block_gas_used BIGINT,
- block_timestamp BIGINT,
- block_transaction_count BIGINT
+ number BIGINT,
+ hash STRING,
+ parent_hash STRING,
+ nonce STRING,
+ sha3_uncles STRING,
+ logs_bloom STRING,
+ transactions_root STRING,
+ state_root STRING,
+ miner STRING,
+ difficulty DECIMAL(38,0),
+ total_difficulty DECIMAL(38,0),
+ size BIGINT,
+ extra_data STRING,
+ gas_limit BIGINT,
+ gas_used BIGINT,
+ timestamp BIGINT,
+ transaction_count BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
diff --git a/schemas/aws/parquet/parquet_blocks.sql b/schemas/aws/parquet/parquet_blocks.sql
index f34434a10..8076a802b 100644
--- a/schemas/aws/parquet/parquet_blocks.sql
+++ b/schemas/aws/parquet/parquet_blocks.sql
@@ -1,21 +1,21 @@
CREATE EXTERNAL TABLE IF NOT EXISTS parquet_blocks (
- block_number BIGINT,
- block_hash STRING,
- block_parent_hash STRING,
- block_nonce STRING,
- block_sha3_uncles STRING,
- block_logs_bloom STRING,
- block_transactions_root STRING,
- block_state_root STRING,
- block_miner STRING,
- block_difficulty DECIMAL(38,0),
- block_total_difficulty DECIMAL(38,0),
- block_size BIGINT,
- block_extra_data STRING,
- block_gas_limit BIGINT,
- block_gas_used BIGINT,
- block_timestamp BIGINT,
- block_transaction_count BIGINT
+ number BIGINT,
+ hash STRING,
+ parent_hash STRING,
+ nonce STRING,
+ sha3_uncles STRING,
+ logs_bloom STRING,
+ transactions_root STRING,
+ state_root STRING,
+ miner STRING,
+ difficulty DECIMAL(38,0),
+ total_difficulty DECIMAL(38,0),
+ size BIGINT,
+ extra_data STRING,
+ gas_limit BIGINT,
+ gas_used BIGINT,
+ timestamp BIGINT,
+ transaction_count BIGINT
)
PARTITIONED BY (start_block BIGINT, end_block BIGINT)
STORED AS PARQUET
diff --git a/schemas/gcp/blocks.json b/schemas/gcp/blocks.json
index 75a440d79..89e6cc2a2 100644
--- a/schemas/gcp/blocks.json
+++ b/schemas/gcp/blocks.json
@@ -1,89 +1,89 @@
[
{
- "name": "block_number",
+ "name": "number",
"type": "INT64",
"mode": "REQUIRED",
"description": "The block number"
},
{
- "name": "block_hash",
+ "name": "hash",
"type": "STRING",
"mode": "REQUIRED",
"description": "Hash of the block"
},
{
- "name": "block_parent_hash",
+ "name": "parent_hash",
"type": "STRING",
"description": "Hash of the parent block"
},
{
- "name": "block_nonce",
+ "name": "nonce",
"type": "STRING",
"mode": "REQUIRED",
"description": "Hash of the generated proof-of-work"
},
{
- "name": "block_sha3_uncles",
+ "name": "sha3_uncles",
"type": "STRING",
"description": "SHA3 of the uncles data in the block"
},
{
- "name": "block_logs_bloom",
+ "name": "logs_bloom",
"type": "STRING",
"description": "The bloom filter for the logs of the block. null when its pending block"
},
{
- "name": "block_transactions_root",
+ "name": "transactions_root",
"type": "STRING",
"description": "The root of the transaction trie of the block"
},
{
- "name": "block_state_root",
+ "name": "state_root",
"type": "STRING",
"description": "The root of the final state trie of the block"
},
{
- "name": "block_miner",
+ "name": "miner",
"type": "STRING",
"description": "The address of the beneficiary to whom the mining rewards were given"
},
{
- "name": "block_difficulty",
+ "name": "difficulty",
"type": "NUMERIC",
"description": "Integer of the difficulty for this block"
},
{
- "name": "block_total_difficulty",
+ "name": "total_difficulty",
"type": "NUMERIC",
"description": "Integer of the total difficulty of the chain until this block"
},
{
- "name": "block_size",
+ "name": "size",
"type": "INT64",
"description": "The size of this block in bytes"
},
{
- "name": "block_extra_data",
+ "name": "extra_data",
"type": "STRING",
"description": "The extra data field of this block"
},
{
- "name": "block_gas_limit",
+ "name": "gas_limit",
"type": "INT64",
"description": "The maximum gas allowed in this block"
},
{
- "name": "block_gas_used",
+ "name": "gas_used",
"type": "INT64",
"description": "The total used gas by all transactions in this block"
},
{
- "name": "block_timestamp",
+ "name": "timestamp",
"type": "INT64",
"description": "The unix timestamp for when the block was collated"
},
{
- "name": "block_transaction_count",
+ "name": "transaction_count",
"type": "INT64",
"description": "The number of transactions in the block"
}
diff --git a/schemas/gcp/transactions_join_receipts.sql b/schemas/gcp/transactions_join_receipts.sql
index a155392b1..a99d10edb 100644
--- a/schemas/gcp/transactions_join_receipts.sql
+++ b/schemas/gcp/transactions_join_receipts.sql
@@ -1,8 +1,8 @@
SELECT
- blocks.block_timestamp,
- TIMESTAMP_SECONDS(blocks.block_timestamp) as block_timestamp_partition,
- blocks.block_hash,
- blocks.block_number,
+ blocks.timestamp,
+ TIMESTAMP_SECONDS(blocks.timestamp) as timestamp_partition,
+ blocks.hash,
+ blocks.number,
transactions.tx_hash,
transactions.tx_nonce,
transactions.tx_index,
@@ -18,5 +18,5 @@ SELECT
receipts.receipt_root,
receipts.receipt_status
FROM `ethereum.blocks` AS blocks
- JOIN `ethereum.transactions` AS transactions ON blocks.block_number = transactions.tx_block_number
+ JOIN `ethereum.transactions` AS transactions ON blocks.number = transactions.tx_block_number
JOIN `ethereum.receipts` AS receipts ON transactions.tx_hash = receipts.receipt_transaction_hash
\ No newline at end of file
diff --git a/tests/resources/test_export_blocks_job/block_with_logs/expected_blocks.csv b/tests/resources/test_export_blocks_job/block_with_logs/expected_blocks.csv
index 6ecfe2e4b..444b15c7e 100644
--- a/tests/resources/test_export_blocks_job/block_with_logs/expected_blocks.csv
+++ b/tests/resources/test_export_blocks_job/block_with_logs/expected_blocks.csv
@@ -1,2 +1,2 @@
-block_number,block_hash,block_parent_hash,block_nonce,block_sha3_uncles,block_logs_bloom,block_transactions_root,block_state_root,block_miner,block_difficulty,block_total_difficulty,block_size,block_extra_data,block_gas_limit,block_gas_used,block_timestamp,block_transaction_count
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count
483920,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,0x2610dc6eb941f4bcbddfd2362b999087ccd956e978f0ece4f8da96851283a2ba,0x57a633e01197dc86,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000800000000000000000000000000000800000000000000000000000000000008000000000000000000000000000000000000021000000080000000004000008000000000000000000000400000000000000000000000000000000400000000000000000000000000000000000000010000000000000000000000000000000000000000400000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000010000000000000000000000000000000000000000000000004000000000000000000000000000000000000040080000,0x2744d46ab0647ed91a9bbd08e19d3bb67491067e8cbe04a276ad2afde5ecd65e,0x48b17dd0031aa97d748a886c912539de22997e861d631fd1eb6509fbabef9651,0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5,7298514125186,2571481026230204460,1113,0xd783010203844765746887676f312e342e32856c696e7578,3141592,143706,1446561880,4
diff --git a/tests/resources/test_export_blocks_job/block_without_transactions/expected_blocks.csv b/tests/resources/test_export_blocks_job/block_without_transactions/expected_blocks.csv
index fd7ccc557..d380ad2cc 100644
--- a/tests/resources/test_export_blocks_job/block_without_transactions/expected_blocks.csv
+++ b/tests/resources/test_export_blocks_job/block_without_transactions/expected_blocks.csv
@@ -1,2 +1,2 @@
-block_number,block_hash,block_parent_hash,block_nonce,block_sha3_uncles,block_logs_bloom,block_transactions_root,block_state_root,block_miner,block_difficulty,block_total_difficulty,block_size,block_extra_data,block_gas_limit,block_gas_used,block_timestamp,block_transaction_count
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count
0,0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3,0x0000000000000000000000000000000000000000000000000000000000000000,0x0000000000000042,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421,0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544,0x0000000000000000000000000000000000000000,17179869184,17179869184,540,0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa,5000,0,0,0
diff --git a/tests/resources/test_export_blocks_job/blocks_with_transactions/expected_blocks.csv b/tests/resources/test_export_blocks_job/blocks_with_transactions/expected_blocks.csv
index 20ed1e98e..fe2dd1667 100644
--- a/tests/resources/test_export_blocks_job/blocks_with_transactions/expected_blocks.csv
+++ b/tests/resources/test_export_blocks_job/blocks_with_transactions/expected_blocks.csv
@@ -1,3 +1,3 @@
-block_number,block_hash,block_parent_hash,block_nonce,block_sha3_uncles,block_logs_bloom,block_transactions_root,block_state_root,block_miner,block_difficulty,block_total_difficulty,block_size,block_extra_data,block_gas_limit,block_gas_used,block_timestamp,block_transaction_count
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count
47218,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0xfc1dd3249585b593ad18822a95873c75779ccbe8a420b457be380b977fc38e87,0xfaa1e51d379b21f7,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xdef39a3afcfb55fabd16d7d5a19d8b078cb95a117ea9596af289525eb8592982,0xab6d1b109c04c1f43f86da33ece31689d9ba1c44ea809d0979c05057a0703c28,0x9746c7e1ef2bd21ff3997fa467593a89cb852bd0,1460233976906,44246932724217368,766,0x476574682f76312e302e312f77696e646f77732f676f312e342e32,42085,42000,1438936285,2
47219,0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0x52cf720359834975,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xd757552351d6a714feda148f8f6f1283e000e05ea407352d45bc9dfc4c16ffe9,0x05a16e52dbacec805dc881439ef54338e8324ee133a4dbbb8ab17f8c73290054,0xf927a40c8b7f6e07c5af7fa2155b4864a4112b13,1459520972035,44248392245189403,763,0x476574682f76312e302e312f6c696e75782f676f312e342e32,42125,42000,1438936326,2