Feature Added: Repo consolidation (hyperledger#215)

* [WIP] Repo merge Merged in sovrin-client, sovrin-common * [WIP] Repo merge Merged in sovrin-client, sovrin-common * [WIP] Repo merge Merged in sovrin-client, sovrin-common * [WIP] Repo merge Merged in sovrin-client, sovrin-common * [WIP] Repo merge Deps fix * [WIP] Repo merge Deps fix * Update Jenkinsfile * Update setup.py
ftsmchl · Jul 10, 2017 · 23bf3b1 · 23bf3b1
1 parent f1ee366
commit 23bf3b1
Show file tree

Hide file tree

Showing 211 changed files with 23,592 additions and 29 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,10 +1,10 @@
 #!groovy
 
-@Library('SovrinHelpers') _
+@Library('SovrinHelpersRepoMerge') _
 
 def name = 'sovrin-node'
 
-def testUbuntu = {
+def nodeTestUbuntu = {
     try {
         echo 'Ubuntu Test: Checkout csm'
         checkout scm
@@ -14,14 +14,11 @@ def testUbuntu = {
 
         testEnv.inside('--network host') {
             echo 'Ubuntu Test: Install dependencies'
-            def sovrinCommon = helpers.extractVersion('sovrin-common')
-            def sovrinClient = helpers.extractVersion('sovrin-client')
-            def plenum = helpers.extractVersionOfSubdependency(sovrinCommon, 'plenum')
-            testHelpers.install(deps: [plenum, sovrinCommon, sovrinClient])
+            testHelpers.install()
 
             echo 'Ubuntu Test: Test'
-            testHelpers.testRunner(resFile: "test-result.${NODE_NAME}.txt")
-            //testHelpers.testJUnit(resFile: "test-result.${NODE_NAME}.xml")
+            testHelpers.testRunner([resFile: "test-result-node.${NODE_NAME}.txt", testDir: 'sovrin_node'])
+            //testHelpers.testJUnit(resFile: "test-result-node.${NODE_NAME}.xml")
         }
     }
     finally {
@@ -30,32 +27,51 @@ def testUbuntu = {
     }
 }
 
-def testWindows = {
-    echo 'TODO: Implement me'
+def clientTestUbuntu = {
+    try {
+        echo 'Ubuntu Test: Checkout csm'
+        checkout scm
+
+        echo 'Ubuntu Test: Build docker image'
+        def testEnv = dockerHelpers.build(name)
+
+        testEnv.inside('--network host') {
+            echo 'Ubuntu Test: Install dependencies'
+            testHelpers.install()
+
+            echo 'Ubuntu Test: Test'
+            testHelpers.testRunner([resFile: "test-result-client.${NODE_NAME}.txt", testDir: 'sovrin_client'])
+            //testHelpers.testJUnit(resFile: "test-result-client.${NODE_NAME}.xml")
+        }
+    }
+    finally {
+        echo 'Ubuntu Test: Cleanup'
+        step([$class: 'WsCleanup'])
+    }
 }
 
-def testWindowsNoDocker = {
+def commonTestUbuntu = {
     try {
-        echo 'Windows No Docker Test: Checkout csm'
+        echo 'Ubuntu Test: Checkout csm'
         checkout scm
 
-        testHelpers.createVirtualEnvAndExecute({ python, pip ->
-            echo 'Windows No Docker Test: Install dependencies'
-            def sovrinClient = helpers.extractVersion('sovrin-client')
-            testHelpers.install(python: python, pip: pip, deps: [sovrinClient], isVEnv: true)
+        echo 'Ubuntu Test: Build docker image'
+        def testEnv = dockerHelpers.build(name)
 
-            echo 'Windows No Docker Test: Test'
-            testHelpers.testJUnit(resFile: "test-result.${NODE_NAME}.xml", python: python)
-        })
+        testEnv.inside {
+            echo 'Ubuntu Test: Install dependencies'
+            testHelpers.install()
+
+            echo 'Ubuntu Test: Test'
+            testHelpers.testJUnit([resFile: "test-result-common.${NODE_NAME}.xml", testDir: 'sovrin_common'])
+        }
     }
     finally {
-        echo 'Windows No Docker Test: Cleanup'
+        echo 'Ubuntu Test: Cleanup'
         step([$class: 'WsCleanup'])
     }
 }
 
-//testAndPublish(name, [ubuntu: testUbuntu, windows: testWindowsNoDocker, windowsNoDocker: testWindowsNoDocker])
-
 options = new TestAndPublishOptions()
 options.enable([StagesEnum.PACK_RELEASE_DEPS, StagesEnum.PACK_RELEASE_ST_DEPS])
-testAndPublish(name, [ubuntu: testUbuntu], true, options)
+testAndPublish(name, [ubuntu: [node: nodeTestUbuntu, client: clientTestUbuntu, common: commonTestUbuntu]], true, options)
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,2 +1,3 @@
 recursive-include data *
+recursive-include sample *
 include post-setup.py
diff --git a/README.md b/README.md
@@ -26,3 +26,37 @@ Have a look at [Setup Instructions](https://github.com/sovrin-foundation/sovrin-
 to understand how to work with the code. Note that setup instructions are
 still changing hour-by-hour as we identify
 install preconditions.
+
+# Indy Client    
+
+[![Build Status](https://jenkins.evernym.com/buildStatus/icon?job=Sovrin%20Client/master)](https://jenkins.evernym.com/job/Sovrin%20Client/job/master/)    
+
+Indy Client provides a command-line tool and underlying interfaces to interact with
+a [self-sovereign identity](https://sovrin.org) ecosystem on top of distributed ledger technology.
+This codebase will eventually be subsumed by (Indy SDK)[https://github.com/hyperledger/indy-sdk);
+if you are looking to build a client or agent, we recommend that you use the SDK instead, because
+the encapsulation and documentation here are not as clean.
+
+[Indy Node](https://github.com/hyperledger/indy-node) is more "core" to Indy; if you want to
+understand how Indy works, starting there may be better.
+
+All bugs, backlog, and stories for Indy (except the SDK) are managed in [Hyperledger's Jira](https://jira.hyperledger.org); use project name INDY.
+
+Developers may want to explore Sovrin's [Getting Started Guide](https://github.com/sovrin-foundation/sovrin-client/blob/master/getting-started.md) to learn about how Indy works. (Sovrin is the public, open ecosystem
+built on top of Indy technology; for more info, see [https://sovrin.org].)
+
+Have a look at [Setup Instructions](https://github.com/sovrin-foundation/sovrin-client/blob/master/setup.md)
+to understand how to work with the code. Note that setup instructions are still changing hour-by-hour as we identify install preconditions.
+
+# Indy Common
+
+[![Build Status](https://jenkins.evernym.com/buildStatus/icon?job=Sovrin%20Common/master)](https://jenkins.evernym.com/view/Core/job/Sovrin%20Common/job/master/)
+
+Common utility functions for other Indy repos (like indy-client, indy-node etc)
+
+This repo will be merged with indy-node at some point soon.
+
+All bugs, backlog, and stories for this repo are managed in [Hyperledger's Jira](https://jira.hyperledger.org).
+
+Join us on [Jira's Rocket.Chat](chat.hyperledger.org) at #indy to discuss.
+
diff --git a/ci/ubuntu.dockerfile b/ci/ubuntu.dockerfile
@@ -33,5 +33,27 @@ USER root
 RUN ln -sf /home/sovrin/test/bin/python /usr/local/bin/python
 RUN ln -sf /home/sovrin/test/bin/pip /usr/local/bin/pip
 USER sovrin
+# TODO: Automate dependency collection
+RUN pip install jsonpickle \
+	ujson \
+	prompt_toolkit==0.57 \
+	pygments \
+	crypto==1.4.1 \
+	rlp \
+	sha3 \
+	leveldb \
+	ioflo==1.5.4 \
+	semver \
+	base58 \
+	orderedset \
+	sortedcontainers==1.5.7 \
+	psutil \
+	pip \
+	portalocker==0.5.7 \
+	pyzmq \
+	raet \
+	ioflo==1.5.4 \
+	psutil \
+	intervaltree
 ENV PYTHONPATH $PYTHONPATH:/home/sovrin/test/bin
 WORKDIR /home/sovrin
diff --git a/cluster-simulation.md b/cluster-simulation.md
@@ -0,0 +1,57 @@
+# Running a Simulation of a Sovrin Cluster and Agents
+One way to run through the [Sovrin Getting Started Guide](getting-started.md) is to set up a simulation of a Sovrin Validator Cluster.  This simulation resides in a single process on a single PC, but it sets up multiple asynchronous call-backs, one for each node being simulated.  These call-backs are handled sequentially in an event loop.  This gives the approximate performance of nultiple Sovrin Validator, Agent and CLI client nodes, but all running within a single process.  These instructions will configure the simulation, leaving you at the end with a CLI command-line prompt that you can use to complete the Getting Started Guide.
+
+## Install the Sovrin Client Software
+
+```
+$ pip install -U --no-cache-dir sovrin-client
+```
+
+If you get any error, check out the info about [prerequisites](https://docs.google.com/document/d/1CyggP4nNPyx4SELNZEc2FOeln6G0F22B37cAVtB_FBM/edit); there are a few dominoes you might have to line up.
+
+
+The install puts some python modules on your system. Most importantly, it gives you a command - line interface(CLI) to Sovrin. We are going to use that CLI to explore what Sovrin can do. (Sovrin also has a programmatic API, but it is not yet fully formalized, and this version of the guide doesn’t document it. See the [Sovrin roadmap](https://github.com/sovrin-foundation/sovrin/wiki/Roadmap).)
+
+## Setup Environment
+We will be doing this exercise in a Python Interactive Console. **_Sovrin must be run with Python 3._** So depending on your system setup, the next command could change.
+
+To launch Python Interactive Console, type this command:
+
+```
+$ python3
+```
+or
+```
+$ python
+```
+
+### Imports
+You now have a Python prompt.  Using the prompt, only one import is required. This import will in turn import other modules and will provide helper functions for this exercise.  Any function with the prefix 'demo' are defined in this import and are just functions to help with the exercise.
+
+```
+>>> from sovrin_client.test.training.getting_started import *
+```
+
+### Run the Sovrin Cluster Simulation
+
+Type this command:
+
+```
+>>> start_getting_started()
+```
+
+This command will start up a local pool of validator "nodes". This can take a few mintues and will produce a lot of console
+output. This output contains the initial communication between 4 nodes. This output can be ignored for this exercise.
+
+After starting up the local sovrin pool, three agents will be launched in virtual "nodes". During this this exercise we will be interacting
+with three agents, Faber Collage, Acme Corp and Thrift Bank. Again, launching these agents can take some time and a lot of
+output.
+
+After these tasks are complete, you should see an interactive prompt, like this:
+
+```
+Sovrin - CLI version 1.17(c) 2016 Evernym, Inc.
+Type 'help' for more information.
+sovrin> 
+```
+You can now proceed with the Getting Started Guide, using this Sovrin client prompt.
diff --git a/data/pool_transactions_live b/data/pool_transactions_live
@@ -0,0 +1,4 @@
+{"data":{"alias":"danube","client_ip":"128.130.204.35","client_port":9722,"node_ip":"128.130.204.35","node_port":9721,"services":["VALIDATOR"]},"dest":"GNEcgS1ZKFLvXbjgjtNJSzQsoYHsBpuCM5UwZC791TPX","identifier":"mBeb2wq5fA3vtdPHCfYTbi9bTBuUQE4ydba9rkQdegz","txnId":"ebf340b317c044d970fcd0ca018d8903726fa70c8d8854752cd65e29d443686c","type":"0"}
+{"data":{"alias":"BIGAWSUSEAST1-001","client_ip":"34.224.255.108","client_port":9796,"node_ip":"34.224.255.108","node_port":9769,"services":["VALIDATOR"]},"dest":"JDCd91PAZr9oEBkwwrhMgG5CGs27HRhCbieLN5H7XBF8","identifier":"6M6oh9Giq88iHRaW7jEj6hksfu99x12GPqNQFU5YLwp8","txnId":"40fceb5fea4dbcadbd270be6d5752980e89692151baf77a6bb64c8ade42ac148","type":"0"}
+{"data":{"alias":"BYU","client_ip":"54.71.209.105","client_port":9722,"node_ip":"54.71.209.105","node_port":9711,"services":["VALIDATOR"]},"dest":"zCxu3oxVGuuKf7rv58wH6hWeUH6gtEjUQ35eUHkXTYt","identifier":"B4paSwxrLVBwaKb7PXMq7EVWeXQvQCwx6HrxUxvuojns","txnId":"ceecfd37686a3ed1759d3cef25e412a800fc8e8846154dbe2a2d72b2af3e3b64","type":"0"}
+{"data":{"alias":"ev1","client_ip":"54.94.255.14","client_port":9702,"node_ip":"54.94.255.14","node_port":9701,"services":["VALIDATOR"]},"dest":"6DAHgFWMDxpXdpAPU6GcSfw6xPEi9fUsH3Z3tFeyicGW","identifier":"DiRwDxvUuNdFCP8kxEWZ51qFyxS4qPjfCPbgEfukvKYU","txnId":"b0c82a3ade3497964cb8034be915da179459287823d92b5717e6d642784c50e6","type":"0"}
diff --git a/data/pool_transactions_local b/data/pool_transactions_local
@@ -0,0 +1,4 @@
+{"data": {"services":["VALIDATOR"], "alias": "Alpha", "client_ip": "127.0.0.1", "client_port": 9702, "node_ip": "127.0.0.1", "node_port": 9701}, "dest": "JpYerf4CssDrH76z7jyQPJLnZ1vwYgvKbvcp16AB5RQ", "identifier": "5rArie7XKukPCaEwq5XGQJnM9Fc5aZE3M9HAPVfMU2xC", "txnId": "d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35", "type": "0"}
+{"data": {"services":["VALIDATOR"], "alias": "Beta", "client_ip": "127.0.0.1", "client_port": 9704, "node_ip": "127.0.0.1", "node_port": 9703}, "dest": "DG5M4zFm33Shrhjj6JB7nmx9BoNJUq219UXDfvwBDPe2", "identifier": "2btLJAAb1S3x6hZYdVyAePjqtQYi2ZBSRGy4569RZu8h", "txnId": "4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce", "type": "0"}
+{"data": {"services":["VALIDATOR"], "alias": "Gamma", "client_ip": "127.0.0.1", "client_port": 9706, "node_ip": "127.0.0.1", "node_port": 9705}, "dest": "AtDfpKFe1RPgcr5nnYBw1Wxkgyn8Zjyh5MzFoEUTeoV3", "identifier": "CECeGXDi6EHuhpwz19uyjjEnsRGNXodFYqCRgdLmLRkt, "txnId": "4b227777d4dd1fc61c6f884f48641d02b4d121d3fd328cb08b5531fcacdabf8a", "type": "0"}
+{"data": {"services":["VALIDATOR"], "alias": "Delta", "client_ip": "127.0.0.1", "client_port": 9708, "node_ip": "127.0.0.1", "node_port": 9707}, "dest": "4yC546FFzorLPgTNTc6V43DnpFrR8uHvtunBxb2Suaa2", "identifier": "3znAGhp6Tk4kmebhXnk9K3jaTMffu82PJfEG91AeRkq2", "txnId": "ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39d", "type": "0"}
diff --git a/data/pool_transactions_sandbox b/data/pool_transactions_sandbox
@@ -0,0 +1,4 @@
+{"dest":"3StSG7YV2Em2Wejf9dA1dNZ9BtZLiUUdm3yJgCS4M8eK","data":{"services":["VALIDATOR"],"client_port":9722,"node_ip":"52.33.22.91","alias":"EvernymV1","client_ip":"52.33.22.91","node_port":9721},"type":"0","txnId":"d4735e3a265e16eee03f59718b9b5d03019c07d8b6c51f90da3a666eec13ab35","identifier":"5rArie7XKukPCaEwq5XGQJnM9Fc5aZE3M9HAPVfMU2xC"}
+{"dest":"13qfUHjTDh3TdRHau93LtkLynwNJnLGoRGh9HQxHuiLs","data":{"services":["VALIDATOR"],"client_port":9724,"node_ip":"52.38.24.189","alias":"EvernymV2","client_ip":"52.38.24.189","node_port":9723},"type":"0","txnId":"4e07408562bedb8b60ce05c1decfe3ad16b72230967de01f640b7e4729b49fce","identifier":"2btLJAAb1S3x6hZYdVyAePjqtQYi2ZBSRGy4569RZu8h"}
+{"dest":"BkJAXQsFYsHtqgbMUYSVQ8n3okRvyPfozwWcn4gV538Z","data":{"services":["VALIDATOR"],"client_port":9730,"node_ip":"34.200.79.65","alias":"RespectNetwork","client_ip":"34.200.79.65","node_port":9729},"type":"0","txnId":"ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe39e","identifier":"4AdS22kC7xzb4bcqg9JATuCfAMNcQYcZa1u5eWzs6cSJ"}
+{"dest":"DNJdyetPD8B6Xsr58PKHynBtEWE2NrMAdnxSMdtgbFER","data":{"services":["VALIDATOR"],"client_port":9747,"node_ip":"52.56.74.57","alias":"BULLDOG","client_ip":"52.56.74.57","node_port":9746},"type":"0","txnId":"ef2d127de37b942baad06145e54b0c619a1f22327b2ebbcfbec78f5564afe3A2","identifier":"6hbecbh36EMK6yAi5NZ9bLZEuRsWFt6qLa2SyMQGXs7H"}
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
@@ -0,0 +1,30 @@
+FROM ubuntu:16.04
+
+MAINTAINER Andrey Zheregelya
+
+# Currently this file contains the copy of validator node but without database config.
+# It also starts CLI as entrypoint by default
+# And also contains tester script
+
+ARG install_sovrin_common="pip3 install --no-cache-dir sovrin-common"
+ARG install_sovrin_client="pip3 install --no-cache-dir sovrin-client"
+
+RUN apt-get update \
+    && apt-get install python3.5 libsodium18 libgmp3-dev flex bison libssl-dev bsdmainutils python3-pip -y \
+    && apt-get install wget -y \
+    && apt-get install git -y \
+    && apt-get clean
+
+# TODO: use native packages when they will be ready.
+RUN wget https://crypto.stanford.edu/pbc/files/pbc-0.5.14.tar.gz \
+    && tar xvf pbc-0.5.14.tar.gz \
+    && cd pbc-0.5.14 \
+    && ./configure && make && make install \
+    && cd .. && rm -rf pbc-0.5.14 pbc-0.5.14.tar.gz
+
+RUN pip3 install --upgrade pip wheel setuptools \
+    && pip3.5 install Charm-Crypto==0.43 \
+    && $install_sovrin_common \
+    && $install_sovrin_client
+
+ENTRYPOINT ["sovrin"]
diff --git a/docker-files/build-sovrin-cli-image.sh b/docker-files/build-sovrin-cli-image.sh
@@ -0,0 +1,59 @@
+#!/bin/bash -x
+
+# Let this helper script be here. But the main helper should be in special ci repo.
+
+# Get parameter of current git repo to download sovrin code
+function construct_repo_string()
+{
+  local package_name="$1"
+  local curr_branch=$(git rev-parse --abbrev-ref HEAD)
+  local curr_commit=$(git rev-parse HEAD)
+  local curr_url=$(git config --get remote.origin.url)
+  local repo_string="git+$curr_url@$curr_commit#egg=$package_name"
+  echo "$repo_string"
+}
+
+package_name="sovrin-client"
+repo_string="git+$curr_url@$curr_commit#egg=$package_name"
+
+do_dev_version=0
+img_tag="sovrin-client-pub"
+folder='./'
+
+while getopts "dr:f:t:c:" opt; do
+  case $opt in
+    d)
+      do_dev_version=1
+      img_tag="sovrin-client-dev"
+      ;;
+    r)
+      repo_string="$OPTARG"
+      ;;
+    f)
+      client_folder="$OPTARG"
+      ;;
+    c)
+      common_folder="$OPTARG"
+      ;;
+    t)
+      img_tag="$OPTARG"
+      ;;
+    *)
+      echo "Undefined option $opt. Exit"
+      exit 1
+      ;;
+  esac
+done
+
+shift $((OPTIND-1))
+
+if [ $do_dev_version -eq 1 ] ; then
+  cd "$client_folder" && client_repo_string=$(construct_repo_string "sovrin_client") ; cd -
+  cd "$common_folder" && common_repo_string=$(construct_repo_string "sovrin_common") ; cd -
+
+  install_sovrin_client="pip3 install --no-cache-dir -e $client_repo_string"
+  install_sovrin_common="pip3 install --no-cache-dir -e $common_repo_string"
+  docker build -t "$img_tag" --build-arg install_sovrin="$install_sovrin_cmd" "$folder"
+else
+  docker build -t "$img_tag" "$folder"
+fi
diff --git a/docker-files/libsodium.key b/docker-files/libsodium.key
@@ -0,0 +1,13 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: SKS 1.1.5
+Comment: Hostname: keyserver.ubuntu.com
+
+mI0ES/EY5AEEAOZl+6Cv7b0fOnXLj8lt1cZiNQHIuOkGRJaMUdvXdrSbtQ4v9GiMWFoFj+9g
+dFN9EjD9JKoXjJb/e/Q9P21uOi0/YmlOfkqWvqm1qsyBXTXTrGx1mghtALPSw0bvYoWZ3aZJ
+3c9VPT5sCdv9IYw6X/+4Z0HoQGvxymbfvRKH3J/xABEBAAG0EkxhdW5jaHBhZCBjaHJpc2xl
+YYi2BBMBAgAgBQJL8RjkAhsDBgsJCAcDAgQVAggDBBYCAwECHgECF4AACgkQuTFqe8eRexLB
+rAQAk9ux3R+k38+dY0f8p3B+0UESy/jNFL/S+t6Fdpw/2qMV1EZohAgJXUw/axmTdr1gKUoy
+GDtE13gebKGy+zqtzsIVo44V0ztC3Z7Kbd9bbiW+wMo7RT4yyi6kURMyE68RrqGbkenZveU6
+o2Urq4LW6bfn5fDLVeYQ5GNsrNdSS1k=
+=9f3N
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/docker-files/sovrin_config.py b/docker-files/sovrin_config.py
@@ -0,0 +1,8 @@
+OrientDB = {
+    "user": "sovrin",
+    "password": "password",
+    "startScript": "/opt/orientdb/bin/server.sh",
+    "shutdownScript": "/opt/orientdb/bin/shutdown.sh"
+}
+EOT
+
diff --git a/docker-files/upload-to-dockerhub.sh b/docker-files/upload-to-dockerhub.sh
@@ -0,0 +1,29 @@
+#!/bin/bash -x
+
+ulogin="$1"
+upassword="$2"
+uimage="$3"
+
+if [ $# -ne 3 ] ; then
+  echo "Incorrect parameters: $0 <user login> <user password> <image to upload>"
+  exit 1
+fi
+
+docker login -u "$ulogin" -p "$upassword"
+ret=$?
+if [ $ret -ne 0 ] ; then
+  echo "Looks like, login was failed. Exit."
+  exit 1
+fi
+
+docker push "$uimage"
+ret=$?
+if [ $ret -ne 0 ] ; then
+  echo "Looks like, image upload was failed. Logout and exit."
+  docker logout
+  exit 1
+fi
+
+echo "Image successfully uploaded"
+docker logout
+exit "$?"
diff --git a/docs/agents-in-sovrin.png b/docs/agents-in-sovrin.png