test script is completed

0.start_blockchain_network.sh

@@ -2,35 +2,50 @@
 
 WORK_DIR=$(pwd)
 
-# stop all nodes
+function getCurTs(){ 
+    cur=$(date +%s.%N)
+
+    cur_s=$(echo $cur | cut -d '.' -f 1)
+    cur_ns=$(echo $cur | cut -d '.' -f 2)
+
+    cur_ms=$(( 10#$cur_s * 1000 + 10#$cur_ns / 1000000 ))
+
+    echo $cur_ms
+}
+
+function printLog(){
+    log_info=$1
+
+    echo -e "\033[34m [SMART][$(getCurTs)] $log_info \033[0m"
+}
+
+## stop all nodes
+printLog "stop existing blockchain nodes"
 bash nodes/127.0.0.1/stop_all.sh
 
-# rm old nodes dir
+## rm old nodes dir
+printLog "remove existing blockchain data"
 rm -rf nodes/
 
-# rm console log
+## rm console log
+printLog "remove existing console log"
 rm -rf console/log/
 rm console/deploylog.txt
 
-# rm pck cache (need root privilege)
-sudo rm /opt/intel/sgx-dcap-pccs/pckcache.db
-sudo rm /opt/intel/sgx-pck-id-retrieval-tool/pckid_retrieval.csv
-
-# restart pccs service (need root privilege)
-sudo systemctl restart pccs
-
-# generate nodes
+## generate nodes
+printLog "generate 4 new blockchain nodes"
 bash build_chain.sh -l 127.0.0.1:4 -p 30300,20200
 
-# start all nodes
+## start all nodes
+printLog "start 4 new blockchain nodes"
 bash nodes/127.0.0.1/start_all.sh
 
-# update conf to console
+## update conf to console
+printLog "sync blockchain nodes config to console"
 cp -r nodes/127.0.0.1/sdk/* console/conf
 
-# start console
-# bash console/start.sh
+## deploy TPM contract
+printLog "deploy TEE provider management contract"
+cd console && bash console.sh deploy TPM
 
-# deploy TPM contract
-echo -e "\033[34m Deploy TPM contract \033[0m"
-cd console && bash console.sh deploy TPM
+printLog "system initialization is completed"

1.register_tee_provider.sh

@@ -2,45 +2,51 @@
 
 WORK_DIR=$(pwd)
 
-function getTiming(){ 
-    start=$1
-    end=$2
+function getCurTs(){ 
+    cur=$(date +%s.%N)
 
-    start_s=$(echo $start | cut -d '.' -f 1)
-    start_ns=$(echo $start | cut -d '.' -f 2)
-    end_s=$(echo $end | cut -d '.' -f 1)
-    end_ns=$(echo $end | cut -d '.' -f 2)
+    cur_s=$(echo $cur | cut -d '.' -f 1)
+    cur_ns=$(echo $cur | cut -d '.' -f 2)
 
-
-    take=$(( ( 10#$end_s - 10#$start_s ) * 1000 + ( 10#$end_ns / 1000000 - 10#$start_ns / 1000000 ) ))
+    cur_ms=$(( 10#$cur_s * 1000 + 10#$cur_ns / 1000000 ))
 
-    echo $take
+    echo $cur_ms
 }
 
-contract_address=$(cat console/deploylog.txt | grep TPM | awk '{print $5}')
+function printLog(){
+    log_info=$1
 
-# retrieve enc_ppid from PCE (need root privilege) and request to attestation service
-start=$(date +%s.%N)
+    echo -e "\033[34m [SMART][$(getCurTs)] $log_info \033[0m"
+}
 
-cd /opt/intel/sgx-pck-id-retrieval-tool/ && sudo ./PCKIDRetrievalTool
+## rm pck cache (need root privilege)
+printLog "remove existing local pck database"
+sudo rm /opt/intel/sgx-dcap-pccs/pckcache.db
+sudo rm /opt/intel/sgx-pck-id-retrieval-tool/pckid_retrieval.csv
 
-end=$(date +%s.%N)
-take=$(getTiming $start $end) 
-echo -e "\033[34m [TIME] It takes ${take} ms for blockchain node to update PCK database \033[0m" # 7722, 6809
+## restart pccs service (need root privilege)
+printLog "restart pck cache server"
+sudo systemctl restart pccs
 
-enc_ppid=$(cat /opt/intel/sgx-pck-id-retrieval-tool/pckid_retrieval.csv | awk -F, '{print $1}')
+printLog "obtain TPM contract address"
+contract_address=$(cat console/deploylog.txt | grep TPM | awk '{print $5}')
 
-# register enc_ppid with tee_provider account in the TPM contract
-echo -e "\033[34m Register TEE provider in the TPM \033[0m"
+## retrieve enc_ppid from PCE (need root privilege) and request to attestation service
+printLog "retrieve enc_ppid from PCE and request pck cache update"
+cd /opt/intel/sgx-pck-id-retrieval-tool/ && sudo ./PCKIDRetrievalTool
 
-start=$(date +%s.%N)
+printLog "obtain enc_ppid"
+enc_ppid=$(cat /opt/intel/sgx-pck-id-retrieval-tool/pckid_retrieval.csv | awk -F, '{print $1}')
 
+## register enc_ppid with tee_provider account in the TPM contract
+printLog "register TEE provider in the TPM contract"
 cd ${WORK_DIR}/console && bash console.sh call TPM $contract_address register_enc_ppid $enc_ppid
 
-end=$(date +%s.%N)
-take=$(getTiming $start $end) 
+printLog "query the latest registered TEE provider"
+bash console.sh call TPM $contract_address get_latest_reg_info
 
-echo -e "\033[34m [TIME] It takes ${take} ms for TEE provider registration \033[0m" # 2426, 2486
+## since the same enc_ppid has been recorded before
+printLog "query an exsited enc_ppid in the pck cache database"
+cd /opt/intel/sgx-pck-id-retrieval-tool/ && sudo ./PCKIDRetrievalTool
 
-echo -e "\033[34m Query registered TEE provider \033[0m"
-bash console.sh call TPM $contract_address get_latest_reg_info
+printLog "TEE provider registration is completed"

2.client_deploy_and_call.sh

@@ -2,40 +2,48 @@
 
 WORK_DIR=$(pwd)
 
-function getTiming(){ 
-    start=$1
-    end=$2
+function getCurTs(){ 
+    cur=$(date +%s.%N)
 
-    start_s=$(echo $start | cut -d '.' -f 1)
-    start_ns=$(echo $start | cut -d '.' -f 2)
-    end_s=$(echo $end | cut -d '.' -f 1)
-    end_ns=$(echo $end | cut -d '.' -f 2)
+    cur_s=$(echo $cur | cut -d '.' -f 1)
+    cur_ns=$(echo $cur | cut -d '.' -f 2)
 
-
-    take=$(( ( 10#$end_s - 10#$start_s ) * 1000 + ( 10#$end_ns / 1000000 - 10#$start_ns / 1000000 ) ))
+    cur_ms=$(( 10#$cur_s * 1000 + 10#$cur_ns / 1000000 ))
 
-    echo $take
+    echo $cur_ms
 }
 
-cd console
+function printLog(){
+    log_info=$1
 
+    echo -e "\033[34m [SMART][$(getCurTs)] $log_info \033[0m"
+}
 
-echo -e "\033[34m Deploy SmartCall contract \033[0m"
-start=$(date +%s.%N)
+cd console
 
+## deploy client smart contract
+printLog "deploy smart contract" 
 contract_address=$(bash console.sh deploy SmartCall | awk 'NR==2' | awk '{print $3}')
 
-end=$(date +%s.%N)
-take=$(getTiming $start $end) 
-echo -e "\033[34m [TIME] It takes ${take} ms to deploy SmartCall contract \033[0m" # 2566 2630 2585
 
+## execute model inference with non-SGX gramine
+printLog "call inference function (non-sgx-gramine, public alexnet model)"
+bash console.sh call SmartCall $contract_address inference "cd /home/junqin/fisco-smart/tee-provider/non_sgx_gramine/alexnet && gramine-direct ./pytorch pytorchexample.py"
+
+## execute model inference with Gramine in SGX
+printLog "call inference function (sgx-gramine, public alexnet model)"
+bash console.sh call SmartCall $contract_address inference "cd /home/junqin/fisco-smart/tee-provider/sgx_gramine/plaintext_model/alexnet && gramine-sgx ./pytorch pytorchexample.py"
+
+## execute private model inference with Gramine in SGX
+printLog "client or TEE provider starts secret key provisioning server"
+cd $WORK_DIR/tee-provider/sgx_gramine/encrypted_model/alexnet/secret_prov_pf && RA_TLS_ALLOW_DEBUG_ENCLAVE_INSECURE=1 RA_TLS_ALLOW_OUTDATED_TCB_INSECURE=1 ./server_dcap wrap_key &
 
+cd $WORK_DIR/console
 
-echo -e "\033[34m Call inference function \033[0m"
-start=$(date +%s.%N)
+printLog "call inference function (sgx-gramine, private alexnet model)"
+bash console.sh call SmartCall $contract_address inference "cd /home/junqin/fisco-smart/tee-provider/sgx_gramine/encrypted_model/alexnet && gramine-sgx ./pytorch pytorchexample.py"
 
-bash console.sh call SmartCall $contract_address inference "cd /home/junqin/fisco-smart/tee-provider && python3 pytorchexample.py"
+printLog "close server_dcap"
+killall server_dcap
 
-end=$(date +%s.%N)
-take=$(getTiming $start $end) 
-echo -e "\033[34m [TIME] It takes ${take} ms to execute model inference \033[0m" # w/o TEE alexnet 3294 3161
+printLog "on-chain and off-chain execution model test is completed"

tee-provider/non_sgx_gramine/alexnet/pytorch.manifest.template

@@ -0,0 +1,70 @@
+# PyTorch manifest template
+
+loader.entrypoint = "file:{{ gramine.libos }}"
+libos.entrypoint = "{{ entrypoint }}"
+
+loader.log_level = "{{ log_level }}"
+
+loader.env.LD_LIBRARY_PATH = "/lib:/usr/lib:{{ arch_libdir }}:/usr/{{ arch_libdir }}"
+
+# Restrict the maximum number of threads to prevent insufficient memory
+# issue, observed on CentOS/RHEL.
+loader.env.OMP_NUM_THREADS = "8"
+
+loader.insecure__use_cmdline_argv = true
+loader.insecure__use_host_env = true
+
+loader.pal_internal_mem_size = "128M"
+
+fs.mounts = [
+  { uri = "file:{{ gramine.runtimedir() }}", path = "/lib" },
+  { uri = "file:{{ arch_libdir }}", path = "{{ arch_libdir }}" },
+  { uri = "file:/usr", path = "/usr" },
+  { uri = "file:/etc", path = "/etc" },
+  { uri = "file:{{ pillow_path }}", path = "{{ pillow_path }}" },
+
+  { type = "tmpfs", path = "/tmp" },
+]
+
+# PyTorch loads its pre-trained models from here
+# Add below uncommented line to fs.mounts array if you want to use torchvision.model.alexnet(pretrained=True)
+# { type = "chroot", uri = "file:{{ env.HOME }}/.cache/torch", path = "{{ env.HOME }}/.cache/torch" }
+
+sgx.nonpie_binary = true
+sgx.enclave_size = "4G"
+sgx.thread_num = 32
+
+sgx.trusted_files = [
+  "file:{{ gramine.libos }}",
+  "file:{{ entrypoint }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:{{ arch_libdir }}/",
+  "file:/usr/{{ arch_libdir }}/",
+  "file:{{ python.stdlib }}/",
+  "file:{{ python.distlib }}/",
+  "file:{{ pillow_path }}",
+  "file:{{ python.get_path('stdlib', vars={'installed_base': '/usr/local'}) }}/",
+
+  "file:pytorchexample.py",
+
+  "file:classes.txt",
+  "file:input.jpg",
+  "file:alexnet-pretrained.pt",  # Pre-trained model saved as a file
+
+  # Uncomment line below if you want to use torchvision.model.alexnet(pretrained=True)
+  # "file:{{ env.HOME }}/.cache/torch/checkpoints/alexnet-owt-4df8aa71.pth",
+]
+
+sgx.allowed_files = [
+  "file:result.txt",
+]
+
+# Gramine optionally provides patched OpenMP runtime library that runs faster
+# inside SGX enclaves (execute `make -C LibOS gcc` to generate it). Uncomment
+# the lines below to use the patched library. PyTorch's SGX perf overhead
+# decreases on some workloads from 25% to 8% with this patched library. Note
+# that we need to preload the library because PyTorch's distribution renames
+# libgomp.so to smth like libgomp-7c85b1e2.so.1, so it's not just a matter of
+# searching in the Gramine's Runtime path first, but a matter of intercepting
+# OpenMP functions.
+# loader.env.LD_PRELOAD = "/lib/libgomp.so.1"

tee-provider/sgx_gramine/encrypted_model/alexnet/Makefile

@@ -0,0 +1,49 @@
+# PyTorch and the pre-trained model must be installed on the system. See README for details.
+
+ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine)
+
+# PyTorch uses the Pillow backend; detect where Pillow was installed
+PILLOW_PATH ?= $(shell pip3 show pillow | grep Location: | cut -d" " -f2)/
+
+ifeq ($(DEBUG),1)
+GRAMINE_LOG_LEVEL = debug
+else
+GRAMINE_LOG_LEVEL = error
+endif
+
+.PHONY: all
+all: pytorch.manifest
+ifeq ($(SGX),1)
+all: pytorch.manifest.sgx pytorch.sig pytorch.token
+endif
+
+pytorch.manifest: pytorch.manifest.template
+	gramine-manifest \
+		-Dlog_level=$(GRAMINE_LOG_LEVEL) \
+		-Dpillow_path=$(PILLOW_PATH) \
+		-Darch_libdir=$(ARCH_LIBDIR) \
+		-Dentrypoint=$(realpath $(shell sh -c "command -v python3")) \
+		$< > $@
+
+# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`),
+# for details on this workaround see
+# https://github.com/gramineproject/gramine/blob/e8735ea06c/CI-Examples/helloworld/Makefile
+pytorch.manifest.sgx pytorch.sig: sgx_sign
+	@:
+
+.INTERMEDIATE: sgx_sign
+sgx_sign: pytorch.manifest
+	gramine-sgx-sign \
+		--manifest $< \
+		--output $<.sgx
+
+pytorch.token: pytorch.sig
+	gramine-sgx-get-token --output $@ --sig $<
+
+.PHONY: clean
+clean:
+	$(RM) *.token *.sig *.manifest.sgx *.manifest
+
+.PHONY: distclean
+distclean: clean
+	$(RM) *.pt result.txt

tee-provider/sgx_gramine/encrypted_model/alexnet/download-pretrained-model.py

@@ -0,0 +1,8 @@
+from torchvision import models
+import torch
+
+output_filename = "alexnet-pretrained.pt"
+alexnet = models.alexnet(pretrained=True)
+torch.save(alexnet, output_filename)
+
+print("Pre-trained model was saved in \"%s\"" % output_filename)