update readme

README.md

@@ -1,33 +1,73 @@
 # GKE + chain-node + ETL
 ---
-## ENV Install (GKE + Chain Node)
-### Use cloud build (recommend)
+A universal chain node on GKE & data collection & analytics solution on GCP!
+
+
+
+## Quickstart
+
+### 1 Initialize your project
 ```
-gcloud init
+gcloud init 
 gcloud auth application-default login
-gcloud config set project <YOUR_PROJECT_ID>
-gcloud config set compute/region <GCP REGION>
+```
+Follow the guide to set up your project id and region. Information needed will be required in the following step.
+We recommend uploading the service account key to manage your project. Please refer to this page for more information https://cloud.google.com/iam/docs/creating-managing-service-account-keys 
+
+### 2 Create secret
+```
+kubectl create secret generic pubsub-key --from-file=key.json=/path/to/your/key/file/
+```
+Create secret file which will be needed when creating pub/sub topic.
+
+### 3 One-shot set up
+```
+./setup-demo.sh
+```
+Once setup, you can run the following command to see the masage published by Pub/Sub.
+```
+python3 subscribe.py
+```
+You may need to set the default credential file as follows if you encounter google.auth.exceptions.DefaultCredentialsError.
+```
+export GOOGLE_APPLICATION_CREDENTIALS='/path/to/your/key/file/‘
+```
 
-cd chain-node
+Or you can run
+```
+kubectl get pods
+kubectl logs $pod_id
+```
+to make sure the full node and etl node are running smooothly.
 
-# Create GKE cluster, add one full node
-./setup.sh
+## Get Started
+For detailed information and additional functions of this project.
 
-# Add full node
+### Add  full node
+If you want to add a new full node to current cluster, please run the following command.
+```
 ./add-node.sh
 ```
+The cluster name and the node name is by default cluster2 and node2 respectively.
 
-### Use terraform to create GKE cluster
+### Add etl node
+If you want to add a node that can listen to the full node and publish information using Pub/Sub, please run the following command.
 ```
-export PROJECT_ID=<YOUR PROJECT ID>
-sed -i'' -e  "s/YOUR_PROJECT_ID/${PROJECT_ID}/g" terraform.tfvars
+./add-etl.sh
+```
+For detailed information about Ethereum-etl, please refer to https://github.com/Tenet-X/ethereum-etl
 
-terraform init
-terraform plan -out=terraform.tfplan
-terraform apply terraform.tfplan
 
-gcloud container clusters get-credentials $(terraform output kubernetes_cluster_name) --region $(terraform output region)
-```
+etl/cloudbuild.yaml will first create a Pub/Sub topic and then apply the deployment.yaml. The secret file created beforehand will be used in this cloudbuild.file.
+Please refer to this page https://cloud.google.com/kubernetes-engine/docs/samples/container-pubsub-secret-deployment
+
+For more information of Pub/Sub deployment with credentials, please visit https://cloud.google.com/kubernetes-engine/docs/samples/container-pubsub-secret-deployment
 
-## GKE + chain Node + ETL
 
+
+### Create snapshot
+If you want to create snapshot for the full node, please set the node name and the cluster name and run the following command.
+```
+./snapshot.sh
+```
+For detailed information about creating snapshot, please refer to https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/volume-snapshots?hl=zh-cn#v1beta1

add-node.sh

@@ -1,18 +1,24 @@
 #!/bin/bash -xe
-read -t 30 -p "Please input GKE cluster name: " CLUSTER_NAME
+CLUSTER_NAME=cluster2
+# read -t 30 -p "Please input GKE cluster name: " CLUSTER_NAME
 echo "GKE cluster name: $CLUSTER_NAME"
 
-read -t 30 -p "Please input chain name: " CHAIN_NAME
-echo "Chain name: $CHAIN_NAME"
-
-read -t 30 -p "Please input full node name: " NODE_NAME
+# read -t 30 -p "Please input chain name: " CHAIN_NAME
+# echo "Chain name: $CHAIN_NAME"
+CHAIN_NAME=geth
+NODE_NAME=node2
+# read -t 30 -p "Please input full node name: " NODE_NAME
 echo "Full node name: $NODE_NAME"
 
+
+
 export CLUSTER_NAME=$CLUSTER_NAME
 export NODE_NAME=$NODE_NAME
+# export SERVICE="$NODE_NAME-service"
 
 PROJECT_ID=$(gcloud config get-value project)
 PROJECT_NUMBER=$(gcloud projects list --filter="$PROJECT_ID" --format="value(PROJECT_NUMBER)")
 GCP_REGION=$(gcloud config get-value compute/region)
-
+# gcloud container clusters get-credentials ${_CLUSTER_NAME} --region ${_GCP_REGION}
+# IP=$(kubectl get service ${SERVICE}--output yaml  | grep 'ip:    ' | sed  "s/- ip:    //g")
 gcloud builds submit --config=add-node/cloudbuild.yaml --substitutions=_PROJECT_ID=${PROJECT_ID},_GCP_REGION=${GCP_REGION},_NODE_NAME=${NODE_NAME},_CLUSTER_NAME=${CLUSTER_NAME},_CHAIN_NAME=${CHAIN_NAME}

etl/cloudbuild.yaml

@@ -22,17 +22,14 @@ steps:
       # IP=$(kubectl get service ${"${_NODE_NAME}-service"}--output yaml  | grep 'ip:    ' | sed  "s/- ip:    //g")
 
       sed -i'' -e  "s/YOUR_NODE_NAME/${_NODE_NAME}/g" etl/deployment.yaml
-      
       sed -i'' -e  "s/YOUR_PROJECT/${_PROJECT_ID}/g" etl/deployment.yaml
-      
-      sed -i'' -e  "s/YOUR_NODE_NAME/${_NODE_NAME}/g" etl/pvc.yaml
-      sed -i'' -e  "s/YOUR_NODE_NAME/${_NODE_NAME}/g" etl/service.yaml
+
       sed -i'' -e  "s/IP_ADDR/$(kubectl get service ${_NODE_NAME}-service --output yaml  |  grep 'ip:' | sed  "s/    - ip: //g")/g" etl/deployment.yaml
-      # kubectl apply -f etl/pvc.yaml
-      # gcloud pubsub topics create crypto_ethereum.blocks
-      # gcloud pubsub subscriptions create crypto_ethereum.blocks.test --topic=crypto_ethereum.blocks
+
+      gcloud pubsub topics create crypto_ethereum.blocks
+      gcloud pubsub subscriptions create crypto_ethereum.blocks.test --topic=crypto_ethereum.blocks
+      # 
       # gcloud pubsub subscriptions create crypto_ethereum.blocks.test --topic=crypto_ethereum.transactions
       kubectl apply -f etl/deployment.yaml
-      # kubectl apply -f etl/service.yaml
       
       

prepare.sh

@@ -1,4 +1,4 @@
 docker pull blockchainetl/ethereum-etl:2
 docker tag blockchainetl/ethereum-etl:2 gcr.io/${PROJECT_ID}/etl
 docker push gcr.io/${PROJECT_ID}/etl
-kubectl create secret generic pubsub-key --from-file=key.json=/Users/litingc/keys.json
+kubectl create secret generic pubsub-key --from-file=key.json=/path-to-your-key-file/

setup-eth.sh

@@ -10,35 +10,17 @@ NODE_NAME=node2
 # read -t 30 -p "Please input full node name: " NODE_NAME
 echo "Full node name: $NODE_NAME"
 
+
+
 export CLUSTER_NAME=$CLUSTER_NAME
 export NODE_NAME=$NODE_NAME
+# export SERVICE="$NODE_NAME-service"
 
 PROJECT_ID=$(gcloud config get-value project)
 PROJECT_NUMBER=$(gcloud projects list --filter="$PROJECT_ID" --format="value(PROJECT_NUMBER)")
 GCP_REGION=$(gcloud config get-value compute/region)
-
-gcloud projects add-iam-policy-binding ${PROJECT_ID} \
-     --member=serviceAccount:${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com \
-     --role roles/iam.serviceAccountTokenCreator
-
-gcloud projects add-iam-policy-binding ${PROJECT_ID} \
-    --member=serviceAccount:${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com \
-     --role roles/container.clusterAdmin
-
-gcloud projects add-iam-policy-binding ${PROJECT_ID} \
-    --member=serviceAccount:${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com \
-     --role roles/container.admin
-
-gcloud projects add-iam-policy-binding ${PROJECT_ID} \
-    --member=serviceAccount:${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com \
-     --role roles/compute.admin
-
-gcloud projects add-iam-policy-binding ${PROJECT_ID} \
-    --member=serviceAccount:${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com \
-     --role roles/iam.serviceAccountUser
-
-gcloud projects add-iam-policy-binding ${PROJECT_ID} \
-    --member=serviceAccount:${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com \
-     --role roles/compute.storageAdmin
+# gcloud container clusters get-credentials ${_CLUSTER_NAME} --region ${_GCP_REGION}
+# IP=$(kubectl get service ${SERVICE}--output yaml  | grep 'ip:    ' | sed  "s/- ip:    //g")
+
 
 gcloud builds submit --config=./cloudbuild-eth.yaml --substitutions=_PROJECT_ID=${PROJECT_ID},_GCP_REGION=${GCP_REGION},_NODE_NAME=${NODE_NAME},_CLUSTER_NAME=${CLUSTER_NAME},_CHAIN_NAME=${CHAIN_NAME}