Remove block_ prefix from CSV and tables

SZNS · Aug 4, 2018 · dc2c3ff · dc2c3ff
1 parent 468b7b4
commit dc2c3ff
Show file tree

Hide file tree

Showing 10 changed files with 111 additions and 111 deletions.
diff --git a/README.md b/README.md
@@ -50,23 +50,23 @@ Read this article https://medium.com/@medvedev1088/exporting-and-analyzing-ether
 
 Column                  | Type               |
 ------------------------|--------------------|
-block_number            | bigint             |
-block_hash              | hex_string         |
-block_parent_hash       | hex_string         |
-block_nonce             | hex_string         |
-block_sha3_uncles       | hex_string         |
-block_logs_bloom        | hex_string         |
-block_transactions_root | hex_string         |
-block_state_root        | hex_string         |
-block_miner             | address            |
-block_difficulty        | numeric            |
-block_total_difficulty  | numeric            |
-block_size              | bigint             |
-block_extra_data        | hex_string         |
-block_gas_limit         | bigint             |
-block_gas_used          | bigint             |
-block_timestamp         | bigint             |
-block_transaction_count | bigint             |
+number            | bigint             |
+hash              | hex_string         |
+parent_hash       | hex_string         |
+nonce             | hex_string         |
+sha3_uncles       | hex_string         |
+logs_bloom        | hex_string         |
+transactions_root | hex_string         |
+state_root        | hex_string         |
+miner             | address            |
+difficulty        | numeric            |
+total_difficulty  | numeric            |
+size              | bigint             |
+extra_data        | hex_string         |
+gas_limit         | bigint             |
+gas_used          | bigint             |
+timestamp         | bigint             |
+transaction_count | bigint             |
 
 ### transactions.csv
 
@@ -445,7 +445,7 @@ Note that NEWLINE_DELIMITED_JSON is used to support REPEATED mode for the column
 Join `transactions` and `receipts`:
 
 ```bash
-> bq mk --table --description "Exported using https://github.com/medvedev1088/ethereum-etl" --time_partitioning_field block_timestamp_partition ethereum.transactions_join_receipts ./schemas/gcp/transactions_join_receipts.json
+> bq mk --table --description "Exported using https://github.com/medvedev1088/ethereum-etl" --time_partitioning_field timestamp_partition ethereum.transactions_join_receipts ./schemas/gcp/transactions_join_receipts.json
 > bq --location=US query --replace --destination_table ethereum.transactions_join_receipts --use_legacy_sql=false "$(cat ./schemas/gcp/transactions_join_receipts.sql | tr '\n' ' ')"
 ```
 

diff --git a/ethereumetl/jobs/exporters/blocks_and_transactions_item_exporter.py b/ethereumetl/jobs/exporters/blocks_and_transactions_item_exporter.py
@@ -24,23 +24,23 @@
 from ethereumetl.jobs.exporters.composite_item_exporter import CompositeItemExporter
 
 BLOCK_FIELDS_TO_EXPORT = [
-    'block_number',
-    'block_hash',
-    'block_parent_hash',
-    'block_nonce',
-    'block_sha3_uncles',
-    'block_logs_bloom',
-    'block_transactions_root',
-    'block_state_root',
-    'block_miner',
-    'block_difficulty',
-    'block_total_difficulty',
-    'block_size',
-    'block_extra_data',
-    'block_gas_limit',
-    'block_gas_used',
-    'block_timestamp',
-    'block_transaction_count'
+    'number',
+    'hash',
+    'parent_hash',
+    'nonce',
+    'sha3_uncles',
+    'logs_bloom',
+    'transactions_root',
+    'state_root',
+    'miner',
+    'difficulty',
+    'total_difficulty',
+    'size',
+    'extra_data',
+    'gas_limit',
+    'gas_used',
+    'timestamp',
+    'transaction_count'
 ]
 
 TRANSACTION_FIELDS_TO_EXPORT = [

diff --git a/ethereumetl/mappers/block_mapper.py b/ethereumetl/mappers/block_mapper.py
@@ -65,21 +65,21 @@ def json_dict_to_block(self, json_dict):
     def block_to_dict(self, block):
         return {
             'type': 'block',
-            'block_number': block.number,
-            'block_hash': block.hash,
-            'block_parent_hash': block.parent_hash,
-            'block_nonce': block.nonce,
-            'block_sha3_uncles': block.sha3_uncles,
-            'block_logs_bloom': block.logs_bloom,
-            'block_transactions_root': block.transactions_root,
-            'block_state_root': block.state_root,
-            'block_miner': block.miner,
-            'block_difficulty': block.difficulty,
-            'block_total_difficulty': block.total_difficulty,
-            'block_size': block.size,
-            'block_extra_data': block.extra_data,
-            'block_gas_limit': block.gas_limit,
-            'block_gas_used': block.gas_used,
-            'block_timestamp': block.timestamp,
-            'block_transaction_count': block.transaction_count,
+            'number': block.number,
+            'hash': block.hash,
+            'parent_hash': block.parent_hash,
+            'nonce': block.nonce,
+            'sha3_uncles': block.sha3_uncles,
+            'logs_bloom': block.logs_bloom,
+            'transactions_root': block.transactions_root,
+            'state_root': block.state_root,
+            'miner': block.miner,
+            'difficulty': block.difficulty,
+            'total_difficulty': block.total_difficulty,
+            'size': block.size,
+            'extra_data': block.extra_data,
+            'gas_limit': block.gas_limit,
+            'gas_used': block.gas_used,
+            'timestamp': block.timestamp,
+            'transaction_count': block.transaction_count,
         }
diff --git a/schemas/aws/blocks.sql b/schemas/aws/blocks.sql
@@ -1,21 +1,21 @@
 CREATE EXTERNAL TABLE IF NOT EXISTS blocks (
-    block_number BIGINT,
-    block_hash STRING,
-    block_parent_hash STRING,
-    block_nonce STRING,
-    block_sha3_uncles STRING,
-    block_logs_bloom STRING,
-    block_transactions_root STRING,
-    block_state_root STRING,
-    block_miner STRING,
-    block_difficulty DECIMAL(38,0),
-    block_total_difficulty DECIMAL(38,0),
-    block_size BIGINT,
-    block_extra_data STRING,
-    block_gas_limit BIGINT,
-    block_gas_used BIGINT,
-    block_timestamp BIGINT,
-    block_transaction_count BIGINT
+    number BIGINT,
+    hash STRING,
+    parent_hash STRING,
+    nonce STRING,
+    sha3_uncles STRING,
+    logs_bloom STRING,
+    transactions_root STRING,
+    state_root STRING,
+    miner STRING,
+    difficulty DECIMAL(38,0),
+    total_difficulty DECIMAL(38,0),
+    size BIGINT,
+    extra_data STRING,
+    gas_limit BIGINT,
+    gas_used BIGINT,
+    timestamp BIGINT,
+    transaction_count BIGINT
 )
 PARTITIONED BY (start_block BIGINT, end_block BIGINT)
 ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'

diff --git a/schemas/aws/parquet/parquet_blocks.sql b/schemas/aws/parquet/parquet_blocks.sql
@@ -1,21 +1,21 @@
 CREATE EXTERNAL TABLE IF NOT EXISTS parquet_blocks (
-    block_number BIGINT,
-    block_hash STRING,
-    block_parent_hash STRING,
-    block_nonce STRING,
-    block_sha3_uncles STRING,
-    block_logs_bloom STRING,
-    block_transactions_root STRING,
-    block_state_root STRING,
-    block_miner STRING,
-    block_difficulty DECIMAL(38,0),
-    block_total_difficulty DECIMAL(38,0),
-    block_size BIGINT,
-    block_extra_data STRING,
-    block_gas_limit BIGINT,
-    block_gas_used BIGINT,
-    block_timestamp BIGINT,
-    block_transaction_count BIGINT
+    number BIGINT,
+    hash STRING,
+    parent_hash STRING,
+    nonce STRING,
+    sha3_uncles STRING,
+    logs_bloom STRING,
+    transactions_root STRING,
+    state_root STRING,
+    miner STRING,
+    difficulty DECIMAL(38,0),
+    total_difficulty DECIMAL(38,0),
+    size BIGINT,
+    extra_data STRING,
+    gas_limit BIGINT,
+    gas_used BIGINT,
+    timestamp BIGINT,
+    transaction_count BIGINT
 )
 PARTITIONED BY (start_block BIGINT, end_block BIGINT)
 STORED AS PARQUET

diff --git a/schemas/gcp/blocks.json b/schemas/gcp/blocks.json
@@ -1,89 +1,89 @@
 [
     {
-        "name": "block_number",
+        "name": "number",
         "type": "INT64",
         "mode": "REQUIRED",
         "description": "The block number"
     },
     {
-        "name": "block_hash",
+        "name": "hash",
         "type": "STRING",
         "mode": "REQUIRED",
         "description": "Hash of the block"
     },
     {
-        "name": "block_parent_hash",
+        "name": "parent_hash",
         "type": "STRING",
         "description": "Hash of the parent block"
     },
     {
-        "name": "block_nonce",
+        "name": "nonce",
         "type": "STRING",
         "mode": "REQUIRED",
         "description": "Hash of the generated proof-of-work"
     },
     {
-        "name": "block_sha3_uncles",
+        "name": "sha3_uncles",
         "type": "STRING",
         "description": "SHA3 of the uncles data in the block"
     },
     {
-        "name": "block_logs_bloom",
+        "name": "logs_bloom",
         "type": "STRING",
         "description": "The bloom filter for the logs of the block. null when its pending block"
     },
     {
-        "name": "block_transactions_root",
+        "name": "transactions_root",
         "type": "STRING",
         "description": "The root of the transaction trie of the block"
     },
     {
-        "name": "block_state_root",
+        "name": "state_root",
         "type": "STRING",
         "description": "The root of the final state trie of the block"
     },
     {
-        "name": "block_miner",
+        "name": "miner",
         "type": "STRING",
         "description": "The address of the beneficiary to whom the mining rewards were given"
     },
     {
-        "name": "block_difficulty",
+        "name": "difficulty",
         "type": "NUMERIC",
         "description": "Integer of the difficulty for this block"
     },
     {
-        "name": "block_total_difficulty",
+        "name": "total_difficulty",
         "type": "NUMERIC",
         "description": "Integer of the total difficulty of the chain until this block"
     },
     {
-        "name": "block_size",
+        "name": "size",
         "type": "INT64",
         "description": "The size of this block in bytes"
     },
     {
-        "name": "block_extra_data",
+        "name": "extra_data",
         "type": "STRING",
         "description": "The extra data field of this block"
     },
     {
-        "name": "block_gas_limit",
+        "name": "gas_limit",
         "type": "INT64",
         "description": "The maximum gas allowed in this block"
     },
     {
-        "name": "block_gas_used",
+        "name": "gas_used",
         "type": "INT64",
         "description": "The total used gas by all transactions in this block"
     },
     {
-        "name": "block_timestamp",
+        "name": "timestamp",
         "type": "INT64",
         "description": "The unix timestamp for when the block was collated"
     },
     {
-        "name": "block_transaction_count",
+        "name": "transaction_count",
         "type": "INT64",
         "description": "The number of transactions in the block"
     }

diff --git a/schemas/gcp/transactions_join_receipts.sql b/schemas/gcp/transactions_join_receipts.sql
@@ -1,8 +1,8 @@
 SELECT
-  blocks.block_timestamp,
-  TIMESTAMP_SECONDS(blocks.block_timestamp) as block_timestamp_partition,
-  blocks.block_hash,
-  blocks.block_number,
+  blocks.timestamp,
+  TIMESTAMP_SECONDS(blocks.timestamp) as timestamp_partition,
+  blocks.hash,
+  blocks.number,
   transactions.tx_hash,
   transactions.tx_nonce,
   transactions.tx_index,
@@ -18,5 +18,5 @@ SELECT
   receipts.receipt_root,
   receipts.receipt_status
 FROM `ethereum.blocks` AS blocks
-  JOIN `ethereum.transactions` AS transactions ON blocks.block_number = transactions.tx_block_number
+  JOIN `ethereum.transactions` AS transactions ON blocks.number = transactions.tx_block_number
   JOIN `ethereum.receipts` AS receipts ON transactions.tx_hash = receipts.receipt_transaction_hash
diff --git a/tests/resources/test_export_blocks_job/block_with_logs/expected_blocks.csv b/tests/resources/test_export_blocks_job/block_with_logs/expected_blocks.csv
@@ -1,2 +1,2 @@
-block_number,block_hash,block_parent_hash,block_nonce,block_sha3_uncles,block_logs_bloom,block_transactions_root,block_state_root,block_miner,block_difficulty,block_total_difficulty,block_size,block_extra_data,block_gas_limit,block_gas_used,block_timestamp,block_transaction_count
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count
 483920,0x246edb4b351d93c27926f4649bcf6c24366e2a7c7c718dc9158eea20c03bc6ae,0x2610dc6eb941f4bcbddfd2362b999087ccd956e978f0ece4f8da96851283a2ba,0x57a633e01197dc86,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000800000000000000000000000000000800000000000000000000000000000008000000000000000000000000000000000000021000000080000000004000008000000000000000000000400000000000000000000000000000000400000000000000000000000000000000000000010000000000000000000000000000000000000000400000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000010000000000000000000000000000000000000000000000004000000000000000000000000000000000000040080000,0x2744d46ab0647ed91a9bbd08e19d3bb67491067e8cbe04a276ad2afde5ecd65e,0x48b17dd0031aa97d748a886c912539de22997e861d631fd1eb6509fbabef9651,0x52bc44d5378309ee2abf1539bf71de1b7d7be3b5,7298514125186,2571481026230204460,1113,0xd783010203844765746887676f312e342e32856c696e7578,3141592,143706,1446561880,4
diff --git a/tests/resources/test_export_blocks_job/block_without_transactions/expected_blocks.csv b/tests/resources/test_export_blocks_job/block_without_transactions/expected_blocks.csv
@@ -1,2 +1,2 @@
-block_number,block_hash,block_parent_hash,block_nonce,block_sha3_uncles,block_logs_bloom,block_transactions_root,block_state_root,block_miner,block_difficulty,block_total_difficulty,block_size,block_extra_data,block_gas_limit,block_gas_used,block_timestamp,block_transaction_count
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count
 0,0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3,0x0000000000000000000000000000000000000000000000000000000000000000,0x0000000000000042,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421,0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544,0x0000000000000000000000000000000000000000,17179869184,17179869184,540,0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa,5000,0,0,0
diff --git a/tests/resources/test_export_blocks_job/blocks_with_transactions/expected_blocks.csv b/tests/resources/test_export_blocks_job/blocks_with_transactions/expected_blocks.csv
@@ -1,3 +1,3 @@
-block_number,block_hash,block_parent_hash,block_nonce,block_sha3_uncles,block_logs_bloom,block_transactions_root,block_state_root,block_miner,block_difficulty,block_total_difficulty,block_size,block_extra_data,block_gas_limit,block_gas_used,block_timestamp,block_transaction_count
+number,hash,parent_hash,nonce,sha3_uncles,logs_bloom,transactions_root,state_root,miner,difficulty,total_difficulty,size,extra_data,gas_limit,gas_used,timestamp,transaction_count
 47218,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0xfc1dd3249585b593ad18822a95873c75779ccbe8a420b457be380b977fc38e87,0xfaa1e51d379b21f7,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xdef39a3afcfb55fabd16d7d5a19d8b078cb95a117ea9596af289525eb8592982,0xab6d1b109c04c1f43f86da33ece31689d9ba1c44ea809d0979c05057a0703c28,0x9746c7e1ef2bd21ff3997fa467593a89cb852bd0,1460233976906,44246932724217368,766,0x476574682f76312e302e312f77696e646f77732f676f312e342e32,42085,42000,1438936285,2
 47219,0x944f09177142833c644c979a83900d8cae1ee67369774b88b3b330bb72825082,0x889c421abc62a48641eee140519e6da8c9dc01d85d8f5c4fbc3c13e3c6e4cb3e,0x52cf720359834975,0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347,0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,0xd757552351d6a714feda148f8f6f1283e000e05ea407352d45bc9dfc4c16ffe9,0x05a16e52dbacec805dc881439ef54338e8324ee133a4dbbb8ab17f8c73290054,0xf927a40c8b7f6e07c5af7fa2155b4864a4112b13,1459520972035,44248392245189403,763,0x476574682f76312e302e312f6c696e75782f676f312e342e32,42125,42000,1438936326,2