Change transactions_join_receipts schema

dendisuhubdy · Jul 3, 2018 · e8575c9 · e8575c9
1 parent 1ecf31d
commit e8575c9
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -317,12 +317,12 @@ To upload CSVs to BigQuery:
 
 ```bash
 > cd ethereum-etl
-> bq --location=US load --source_format=CSV --skip_leading_rows=1 ethereum.blocks gs://<your_bucket>/ethereumetl/export/blocks/*.csv ./schemas/gcp/blocks.json
-> bq --location=US load --source_format=CSV --skip_leading_rows=1 ethereum.transactions gs://<your_bucket>/ethereumetl/export/transactions/*.csv ./schemas/gcp/transactions.json
-> bq --location=US load --source_format=CSV --skip_leading_rows=1 --max_bad_records=5000 ethereum.erc20_transfers gs://<your_bucket>/ethereumetl/export/erc20_transfers/*.csv ./schemas/gcp/erc20_transfers.json
-> bq --location=US load --source_format=CSV --skip_leading_rows=1 ethereum.receipts gs://<your_bucket>/ethereumetl/export/receipts/*.csv ./schemas/gcp/receipts.json
-> bq --location=US load --source_format=CSV --skip_leading_rows=1 ethereum.logs gs://<your_bucket>/ethereumetl/export/logs/*.csv ./schemas/gcp/logs.json
-> bq --location=US load --source_format=CSV --skip_leading_rows=1 ethereum.contracts gs://<your_bucket>/ethereumetl/export/contracts/*.csv ./schemas/gcp/contracts.json
+> bq --location=US load --replace --source_format=CSV --skip_leading_rows=1 ethereum.blocks gs://<your_bucket>/ethereumetl/export/blocks/*.csv ./schemas/gcp/blocks.json
+> bq --location=US load --replace --source_format=CSV --skip_leading_rows=1 ethereum.transactions gs://<your_bucket>/ethereumetl/export/transactions/*.csv ./schemas/gcp/transactions.json
+> bq --location=US load --replace --source_format=CSV --skip_leading_rows=1 --max_bad_records=5000 ethereum.erc20_transfers gs://<your_bucket>/ethereumetl/export/erc20_transfers/*.csv ./schemas/gcp/erc20_transfers.json
+> bq --location=US load --replace --source_format=CSV --skip_leading_rows=1 ethereum.receipts gs://<your_bucket>/ethereumetl/export/receipts/*.csv ./schemas/gcp/receipts.json
+> bq --location=US load --replace --source_format=CSV --skip_leading_rows=1 ethereum.logs gs://<your_bucket>/ethereumetl/export/logs/*.csv ./schemas/gcp/logs.json
+> bq --location=US load --replace --source_format=CSV --skip_leading_rows=1 ethereum.contracts gs://<your_bucket>/ethereumetl/export/contracts/*.csv ./schemas/gcp/contracts.json
 ```
 
 Note that `--max_bad_records` is needed for erc20_transfers to avoid 

diff --git a/schemas/gcp/transactions_join_receipts.json b/schemas/gcp/transactions_join_receipts.json
@@ -2,21 +2,23 @@
     {
         "name": "block_timestamp",
         "type": "INT64",
+        "mode": "REQUIRED",
         "description": "The unix timestamp for when the block was collated"
     },
     {
         "name": "block_timestamp_partition",
         "type": "TIMESTAMP",
+        "mode": "REQUIRED",
         "description": "The timestamp for when the block was collated"
     },
     {
-        "name": "tx_block_hash",
+        "name": "block_hash",
         "type": "STRING",
         "mode": "REQUIRED",
         "description": "Hash of the block where this transaction was in. null when its pending"
     },
     {
-        "name": "tx_block_number",
+        "name": "block_number",
         "type": "INT64",
         "mode": "REQUIRED",
         "description": "Block number where this transaction was in. null when its pending"

diff --git a/schemas/gcp/transactions_join_receipts.sql b/schemas/gcp/transactions_join_receipts.sql
@@ -1,8 +1,8 @@
 SELECT
   blocks.block_timestamp,
   TIMESTAMP_SECONDS(blocks.block_timestamp) as block_timestamp_partition,
-  transactions.tx_block_hash,
-  transactions.tx_block_number,
+  blocks.block_hash,
+  blocks.block_number,
   transactions.tx_hash,
   transactions.tx_nonce,
   transactions.tx_index,