merge master

Signed-off-by: ashcherbakov <[email protected]>

Jenkinsfile.cd

@@ -5,6 +5,7 @@
 String name = 'indy-node'
 String pkgName = name
 String mainModuleName = 'indy_node'
+Boolean gatherLogs = (params.GATHER_LOGS ?: env.GATHER_LOGS) != 'false'
 
 def nodeTestUbuntu = {
     try {
@@ -91,8 +92,9 @@ def systemTests = { component, releaseVersion ->
             ['test_vc.py'],
             ['test_consensus.py']
         ]
-        testVersion = 'v0.8.4'
+        testVersion = 'v0.8.8'
         testVersionByTag = true
+        delegate.gatherLogs = gatherLogs
     }
 }
 

Jenkinsfile.nightly

@@ -5,6 +5,7 @@
 String pkgName = 'indy-node'
 String mainModuleName = 'indy_node'
 String emailRecipients = params.INDY_NODE_RECIPIENTS ?: env.INDY_NODE_RECIPIENTS ?: ''
+Boolean gatherLogs = (params.GATHER_LOGS ?: env.GATHER_LOGS) != 'false'
 
 def localLib
 def err
@@ -47,10 +48,21 @@ try {
             ['test_consensus.py', 'TestTAASuite.py'],
             ['test_upgrade.py', 'test_roles.py', 'test_freshness.py', 'TestMultiSigSuite.py'],
             ['TestAuditSuite.py'],
-            ['TestAuthMapSuite.py']
+            // set of authmap tests
+            // TODO might be groupped in parts once https://github.com/docker/docker-py/issues/2278 is resolved
+            ['TestAuthMapAttribSuite.py'],
+            ['TestAuthMapCredDefSuite.py'],
+            ['TestAuthMapMiscSuite.py'],
+            ['TestAuthMapNymSuite.py'],
+            ['TestAuthMapPluginsSuite.py'],
+            ['TestAuthMapRevocRegDefSuite.py'],
+            ['TestAuthMapRevocRegEntrySuite.py'],
+            ['TestAuthMapSchemaSuite.py'],
+            ['TestAuthMapUpgradeSuite.py']
         ]
-        testVersion = 'v0.8.4'
+        testVersion = 'v0.8.8'
         testVersionByTag = true
+        delegate.gatherLogs = gatherLogs
     }
 } catch(Exception _err) {
     currentBuild.result = "FAILED"

ci/pipeline.groovy

@@ -12,7 +12,8 @@ def systemTests(Closure body) {
             srcVersion: null,
             testSchema: [['.']],
             testVersion: null,
-            testVersionByTag: false
+            testVersionByTag: false,
+            gatherLogs: true
         ],
         body, ['pkgVersion'], {}, prefix
     )
@@ -92,14 +93,17 @@ def systemTests(Closure body) {
                 String testReportFileNameXml = "system_tests_${testGroup}_report.${config.repoChannel}.xml"
                 String testReportFileNamePlain = "system_tests_${testGroup}_report.${config.repoChannel}.txt"
                 String testTargets = config.testSchema[testGroup].collect{"system/indy-node-tests/$it"}.join(' ')
+                String buildLogsDir = "_build/logs"
+                String gatherLogsOpt = config.gatherLogs ? ' --gatherlogs' : ''
+
                 try {
                     stage("[${testGroup}] Run tests") {
                         sh """
                             bash -c "\
                                 set -o pipefail; \
                                 ./system/docker/run.sh \
                                     \\"$testTargets\\" \
-                                    \\"-l -vv --junit-xml=$testReportFileNameXml\\" \
+                                    \\"-l -vv --junit-xml=$testReportFileNameXml ${gatherLogsOpt} --logsdir=${buildLogsDir}\\" \
                                     \\"$systemTestsNetwork\\" 2>&1 | tee $testReportFileNamePlain;\
                             "
                         """
@@ -112,6 +116,7 @@ def systemTests(Closure body) {
                         sh "ls -la *report* || true"
                         if (err) {
                             archiveArtifacts artifacts: testReportFileNamePlain, allowEmptyArchive: true
+                            archiveArtifacts artifacts: "$buildLogsDir/**/*", allowEmptyArchive: true
                         }
                         junit testResults: testReportFileNameXml, allowEmptyResults: true
                     }
@@ -173,28 +178,24 @@ def systemTests(Closure body) {
                 error "Failed to get versions for indy-plenum or indy-crypto or indy-sdk"
             }
         }
+    }
 
-        Map builds = [:]
-        for (int i = 0; i < config.testSchema.size(); i++) {
-            String testNames = config.testSchema[i].join(' ')
-            Boolean isFirst = (i == 0)
-            int testGroup = i
-            builds[testNames] = {
-                stage("Run ${testNames}") {
-                    if (isFirst) {
-                        runTest(testGroup)
-                    } else {
-                        nodeWrapper('ubuntu') {
-                            runTest(testGroup)
-                        }
-                    }
+    Map builds = [:]
+    for (int i = 0; i < config.testSchema.size(); i++) {
+        String testNames = config.testSchema[i].join(' ')
+        Boolean isFirst = (i == 0)
+        int testGroup = i
+        builds[testNames] = {
+            stage("Run ${testNames}") {
+                nodeWrapper('ubuntu') {
+                    runTest(testGroup)
                 }
             }
         }
-        builds.failFast = false
-
-        parallel builds
     }
+    builds.failFast = false
+
+    parallel builds
 }
 
 return this;

design/transaction_endorser.md

@@ -0,0 +1,75 @@
+# Transaction Endorser Design
+## Reasoning
+As a transaction author, I need my transactions to be written to the ledger preserving me as the author without my needing to accept the responsibilities of an endorser so that I can focus on my business. Instead, I will have a business relationship with an endorser who will endorse my transactions.
+
+## Requirements
+
+ - It is easy to tell from the ledger who is the endorser and who is the transaction author for each transaction.
+ - A transaction author can use a different transaction endorser for future transactions, including updates to attribs and key rotations.
+ - The transaction must use the author key to sign the transaction author agreement 
+ - If the endorser field is included in a transaction, then the ledger will reject the transaction if it is not signed by the endorser.
+
+## Proposed workflow
+1. Transaction Author builds a new request (`indy_build_xxx_reqeust`).
+1. If no endorser is needed for a transaction (for example, the transaction author is an endorser, or auth rules are configured in a way that transaction author can send requests in permissionless mode), then the author signs and submits the transaction.
+1. Otherwise the author chooses an Endorser and adds Endorser's DID into the request calling `indy_append_request_endorser`. 
+1. Transaction author signs the request (`indy_multi_sign_request` or `indy_sign_request`) with the added endorser field (output of `indy_append_request_endorser`).
+1. Transaction author sends the request to the Endorser (out of scope).
+1. Transaction Endorser signs the request (as of now `indy_multi_sign_request` must be called, not `indy_sign_request`) and submits it to the ledger.
+
+## Assumption
+Transaction Author must have a DID on the ledger. The only possible exception is a creation of new DIDs (NYMs) in permissionless mode if it's allowed by the corresponding auth rule. 
+
+## Changes in Write Request format
+
+- A new optional `endorser` field will be added to every write request. It points to a DID of the transaction endorser. 
+- An existing `identifier` field points to a DID of the original author.
+- If there is no `endorser` field, then it's assumed that the original author (`identifier`) is the endorser. 
+
+## libindy API to support endorsers
+```rust=
+/// Append Endorser to existing request expecting that the transaction will be sent by the specified Endorser.
+///
+///
+/// #Params
+/// request_json: original request
+/// endorser_did: DID of the Endorser that will submit the transaction. 
+///                         The Endorser's DID must be present on the ledger.
+/// cb: Callback that takes command result as parameter. 
+///     The command result is a request JSON with Endorser field appended.
+#[no_mangle]
+pub extern fn indy_append_request_endorser(command_handle: CommandHandle,
+                                           request_json:*const c_char,
+                                           endorser_did: *const c_char,
+                                           cb: Option<extern fn(command_handle_: CommandHandle,
+                                                                err: ErrorCode,
+                                                                request_json: *const c_char)>) -> ErrorCode
+```
+
+
+
+## Changes on Indy-Node side
+
+#### General Idea
+- Transaction author's DID is `identifier` field, and it's used the same way as of now. So, it's used for identification of transactions from this author.
+- Endorser's DID is put into an optional `endorser` field. If this field is present, then signatures from the both Author and Endorser are expected.
+
+
+#### Request static validation
+- If there is `endorser` field, then 
+  - `signatures` must be present
+  - there must be `endorser`'s signature in `signatures`
+  - there must be `identifier`'s signature in `signatures`
+  - `signature` must be absent
+
+#### Signature Verification
+No changes are required. 
+
+Since we check in Request static validation that a `signatures` field containing signatures from the both `endorser` and `identifier` must be present in Endorser case, it's sufficient just to verify signatures by a common logic.
+
+
+#### Auth Rules
+No changes are required.
+- `is_owner` must always be checked against `identifier` field (this is how it is now).
+- number of signatures of expected roles is checked against `signature` or `signatures` by common logic. Since Endorser's signature is there, existing common logic will work. 
+

docs/source/ci-cd.md

@@ -127,36 +127,84 @@ Each `build-scripts` folder includes `Readme.md`. Please check them for more det
 
 ## Release workflow
 
-1. Release candidate preparation
-    1. [**Maintainer**] Creates a new release branch `release-X.Y.Z` based on `stable`.
-    2. [**Contributor**]
-        - Creates a new release candidate branch (e.g. `rc-X.Y.Z.rc1`) based on that release branch.
-        - Merges `master` branch.
-        - Sets stable version of `indy-plenum` in `setup.py` (for `indy-node` only).
-        - Sets new version `X.Y.Z.rc1` (`./bump_version.sh X.Y.Z.rc1`).
-        - Commits and pushes changes.
-        - Creates a release candidate PR to `release-X.Y.Z`.
-    3. [**Maintainer**] Waits for CI, reviews the release candidate PR and either merges the PR or asks for changes.
-2. Release candidate acceptance
-    1. [**Maintainer**] Once the release candidate PR is merged the maintainer **starts release candidate pipeline manually**.
-    2. [**build server**] Once the CD pipeline is started (manually triggered) for branch `release-X.Y.Z` it does the following:
-        - creates and pushes release commit to `release-X.Y.Z`;
-        - publishes release candidates packages to PyPI and debian `rc` components;
-        - performs system testing (`indy-node` only);
-        - creates a release PR to merge `release-X.Y.Z` to `stable`;
-        - waits for an approval to proceed.
-    3. [**Maintainer/QA**] Waits for CI, reviews the release PR and either approves or rejects:
-        - may run additional tests against the release candidate before approval;
-        - in case of approval lets build server to proceed but **does not merge the release PR manually**;
-        - otherwise stops the pipeline and previous steps are repeated for new release candidate `X.Y.Z.rc1` and possible future ones.
-    4. [**build server**]
-        - once it is approved to proceed performs fast-forward merging to stable and creates tag `vX.Y.Z`;
-        - otherwise rollbacks release commit pushed to release branch `release-X.Y.Z`.
-3. Publishing
-    1. [**build server**] Once the release PR is merged stable pipeline is triggered and it:
-        - publishes to PyPI;
-        - re-packs `rc` debian package and publishes to debian `stable` components.
-
-Hotfix releases are quite similar except the following difference:
-  - hotifx branches `hotfix-X.Y.Z` are created from git tag `vX.Y.(Z-1)`;
-  - `master` is not merged since hotfixes (as a rule) should include only fixes for stable code.
+### Feature Release
+
+#### 1. Release Candidate Preparation
+
+1. [**Maintainer**]
+    - Create `release-X.Y.Z` branch from `stable` (during the first RC preparation only).
+2. [**Contributor**]
+    - Create `rc-X.Y.Z.rcN` branch from `release-X.Y.Z` (`N` starts from `1` and is incremented for each new RC).
+    - Apply necessary changes from `master` (either `merge` or `cherry-pick`).
+    - (_optional_) [`indy-node`] Set `indy-plenum` version in `setup.py`.
+    - Set the package version `./bump_version.sh X.Y.Z.rcN`.
+    - Commit, push and create a PR to `release-X.Y.Z`.
+3. Until PR is merged:
+    1. [**build server**]
+        - Run CI for the PR and notifies GitHub.
+    2. [**Maintainer**]
+        - Review the PR.
+        - Either ask for changes or merge.
+    3. [**Contributor**]
+        - (_optional_) Update the PR if either CI failed or reviewer asked for changes.
+        - (_optional_) [**indy-node**] Bump `indy-plenum` version in `setup.py` if changes require new `indy-plenum` release.
+
+#### 2. Release Candidate Acceptance
+
+**Note** If any of the following steps fails new release candidate should be prepared.
+
+1. [**Maintainer**]
+    - **Start release candidate pipeline manually**.
+2. [**build server**]
+    - Checkout the repository.
+    - Publish to PyPI as `X.Y.Z.rcN`.
+    - Bump version locally to `X.Y.Z`, commit and push as the `release commit` to remote.
+    - Build debian packages:
+        - for the project: source code version would be `X.Y.Z`, debian package version `X.Y.Z~rcN`;
+        - for the 3rd party dependencies missed in the official debian repositories.
+    - Publish the packages to `rc-latest` debian channel.
+    - [`indy-node`] Copy the package along with its dependencies (including `indy-plenum`)
+      from `rc-latest` to `rc` channel.
+    - [`indy-node`] Run system tests for the `rc` channel.
+    - Create **release PR** from `release-X.Y.Z` (that points to `release commit`) branch to `stable`.
+    - Notify maintainers.
+    - Wait for an approval to proceed. **It shouldn't be provided until `release PR` passes all necessary checks** (e.g. DCO, CI testing, maintainers reviews etc.).
+3. [**build server**]
+    - Run CI for the PR and notify GitHub.
+4. [**QA**]
+    - (_optional_) Perform additional testing.
+5. [**Maintainer**]
+    - Review the PR but **do not merge it**.
+    - If approved: let build server to proceed.
+    - Otherwise: stop the pipeline.
+6. [**build server**]
+    - If approved:
+        - perform fast-forward merge;
+        - create and push tag `vX.Y.Z`;
+        - Notify maintainers.
+    - Otherwise rollback `release commit` by moving `release-X.Y.Z` to its parent.
+
+#### 3. Publishing
+
+1. [**build server**] triggered once the `release PR` is merged
+    - Publish to PyPI as `X.Y.Z`.
+    - Download and re-pack debian package `X.Y.Z~rcN` (from `rc-latest` channel) to `X.Y.Z` changing only the package name.
+    - Publish the package to `rc-latest` debian channel.
+    - Copy the package along with its dependencies from `rc-latest` to `stable-latest` channel.
+    - [`indy-node`] Copy the package along with its dependencies (including `indy-plenum`) from `stable-latest` to `stable` channel.
+    - [`indy-node`] Run system tests for the `stable` channel.
+    - Notify maintainers.
+
+#### 4. New Development Cycle Start
+
+1. [**Contributor**]:
+    - Create PR to `master` with version bump to `X'.Y'.Z'.dev0`, where `X'.Y'.Z'` is next target release version. Usually it increments one of `X`, `Y` or `Z` and resets lower parts (check [SemVer](https://semver.org/) for more details), e.g.:
+        - `X.Y.Z+1` - bugfix release
+        - `X.Y+1.0` - feature release, backwards compatible API additions/changes
+        - `X+1.0.0` - major release, backwards incompatible API changes
+
+### Hotfix Release
+
+Hotfix release is quite similar except the following difference:
+  - hotfix branches named `hotfix-X.Y.Z`;
+  - `master` usually is not merged since hotfixes (as a rule) should include only fixes for stable code.

indy_common/authorize/auth_actions.py

@@ -1,14 +1,13 @@
 from typing import NamedTuple
 
-
 from abc import ABCMeta, abstractmethod
 
-
 RULE_DELIMETER = "--"
 ADD_PREFIX = "ADD"
 EDIT_PREFIX = "EDIT"
 
-ActionDef = NamedTuple('ActionDef', [('txn_type', str), ('prefix', str), ('field', str), ('old_value', str), ('new_value', str)])
+ActionDef = NamedTuple('ActionDef',
+                       [('txn_type', str), ('prefix', str), ('field', str), ('old_value', str), ('new_value', str)])
 
 
 def compile_action_id(txn_type,
@@ -44,7 +43,11 @@ def get_action_id(self) -> str:
 
 
 class AuthActionAdd(AbstractAuthAction):
-    def __init__(self, txn_type, field=None, value=None, is_owner: bool=True):
+    def __init__(self,
+                 txn_type,
+                 field=None,
+                 value=None,
+                 is_owner: bool = True):
         self.txn_type = txn_type
         self.field = str(field) if field is not None else ''
         self.value = str(value) if value is not None else ''
@@ -64,7 +67,7 @@ def __init__(self,
                  field=None,
                  old_value=None,
                  new_value=None,
-                 is_owner: bool=True):
+                 is_owner: bool = True):
         self.txn_type = txn_type
         self.field = str(field) if field is not None else ''
         self.old_value = str(old_value) if old_value is not None else ''