copies similar activation changes from plenum

Signed-off-by: Andrey Kononykhin <[email protected]>

Jenkinsfile.cd

@@ -1,8 +1,10 @@
 #!groovy
 
-@Library('[email protected]') _
+// TODO doesn't exist yet
+@Library('[email protected]') _
 
-def name = 'indy-node'
+String name = 'indy-node'
+String pkgName = name
 
 def nodeTestUbuntu = {
     try {
@@ -49,17 +51,25 @@ def commonTestUbuntu = {
     }
 }
 
-def buildDebUbuntu = { repoName, releaseVersion, sourcePath ->
+def buildDebUbuntu = { releaseVersion, sourcePath, packageVersion=null, missedPkgs=false ->
     def volumeName = "$name-deb-u1604"
+    packageVersion = packageVersion ?: releaseVersion
+
     if (env.BRANCH_NAME != '' && env.BRANCH_NAME != 'master') {
         volumeName = "${volumeName}.${BRANCH_NAME}"
     }
     if (sh(script: "docker volume ls -q | grep -q '^$volumeName\$'", returnStatus: true) == 0) {
         sh "docker volume rm $volumeName"
     }
+
+    // TODO build only missed ones
     dir('build-scripts/ubuntu-1604') {
-        sh "./build-$name-docker.sh \"$sourcePath\" $releaseVersion $volumeName"
-        sh "./build-3rd-parties-docker.sh $volumeName"
+        sh "./build-$name-docker.sh \"$sourcePath\" $releaseVersion $volumeName $packageVersion"
+        if (missedPkgs == [pkgName]) {
+            echo "Skip 3rd parties building"
+        } else {
+            sh "./build-3rd-parties-docker.sh $volumeName"
+        }
     }
     return "$volumeName"
 }
@@ -106,7 +116,7 @@ def systemTests = { component, releaseVersion ->
 
             stage("[${testFileName}] Ensure node services started") {
                 // TODO explore and fix the reason why they (mostly Node1) not always active
-                for (nodeNum = 1; nodeNum <= nodesNum; nodeNum++) {
+                for (int nodeNum = 1; nodeNum <= nodesNum; nodeNum++) {
                     for (service in ['indy-node', 'indy-node-control']) {
                         sh """
                             for i in 1 2 3; do
@@ -262,7 +272,7 @@ def systemTests = { component, releaseVersion ->
 
             Map builds = [:]
             List tests = ['test_ledger.py', 'test_consensus.py', 'test_vc.py']
-            for (i = 0; i < tests.size(); i++) {
+            for (int i = 0; i < tests.size(); i++) {
                 String testFileName = tests[i]
                 Boolean isFirst = (i == 0)
                 builds[testFileName] = {
@@ -295,10 +305,27 @@ def systemTests = { component, releaseVersion ->
     }
 }
 
-options = new TestAndPublishOptions()
+def options = new TestAndPublishOptions()
+// TODO review that
+options.setApprovers(['QA'])
+options.setPkgName(pkgName)
+
+// TODO duplicates list from build scripts
+options.setPkgDeps([
+    'python3-timeout-decorator': '0.4.0',
+    'python3-distro': '1.3.0',
+])
+
+
 options.enable([StagesEnum.PACK_RELEASE_COPY, StagesEnum.PACK_RELEASE_COPY_ST])
 options.setCopyWithDeps(true)
 options.setSystemTestsCb(systemTests)
 options.setPrContexts([env.INDY_GITHUB_PR_REQUIRED_CONTEXT ?: "ci/hyperledger-jenkins/pr-merge"])
 
-testAndPublish(name, [ubuntu: [node: nodeTestUbuntu, common: commonTestUbuntu]], true, options, [ubuntu: buildDebUbuntu])
+testAndPublish(
+    name,
+    [
+        ubuntu: [node: nodeTestUbuntu, common: commonTestUbuntu]
+    ],
+    true, options, [ubuntu: buildDebUbuntu], 'indy_node'
+)

MANIFEST.in

@@ -1,3 +1,5 @@
 recursive-include data *
 recursive-include sample *
 include post-setup.py
+include indy_node/__version__.json
+include indy_node/__manifest__.json

build-scripts/ubuntu-1604/build-3rd-parties.sh

@@ -40,6 +40,6 @@ function build_from_pypi {
 
 # build 3rd parties:
 #   build_from_pypi <pypi-name> <version>
-
+# TODO duplicates list from Jenkinsfile.cd
 build_from_pypi timeout-decorator 0.4.0
 build_from_pypi distro 1.3.0

build-scripts/ubuntu-1604/build-indy-node-docker.sh

@@ -5,16 +5,17 @@ VERSION="$2"
 PKG_NAME=indy-node
 IMAGE_NAME="${PKG_NAME}-build-u1604"
 OUTPUT_VOLUME_NAME="${3:-"${PKG_NAME}-deb-u1604"}"
+PACKAGE_VERSION="${4:-$VERSION}"
 
 if [[ (-z "${PKG_SOURCE_PATH}") || (-z "${VERSION}") ]]; then
-    echo "Usage: $0 <path-to-package-sources> <version> <volume>"
+    echo "Usage: $0 <path-to-package-sources> <version> [<volume> [package-version]]"
     exit 1;
 fi
 
-if [ -z "$4" ]; then
-    CMD="/root/build-${PKG_NAME}.sh /input ${VERSION} /output"
+if [ -z "$5" ]; then
+    CMD="/root/build-${PKG_NAME}.sh /input ${VERSION} /output ${PACKAGE_VERSION}"
 else
-    CMD="$4"
+    CMD="$5"
 fi
 
 docker build -t "${IMAGE_NAME}" -f Dockerfile .
@@ -28,4 +29,3 @@ docker run \
     -e PKG_NAME="${PKG_NAME}" \
     "${IMAGE_NAME}" \
     $CMD
-

build-scripts/ubuntu-1604/build-indy-node.sh

@@ -3,6 +3,7 @@
 INPUT_PATH="$1"
 VERSION="$2"
 OUTPUT_PATH="${3:-.}"
+PACKAGE_VERSION=${4:-$VERSION}
 
 PACKAGE_NAME=indy-node
 
@@ -34,6 +35,7 @@ fpm --input-type "python" \
     --after-install "postinst_node" \
     --before-remove "prerm" \
     --name "${PACKAGE_NAME}" \
+    --version ${PACKAGE_VERSION} \
     --package "${OUTPUT_PATH}" \
     "${TMP_DIR}"
 

build-scripts/ubuntu-1604/prepare-package.sh

@@ -8,57 +8,36 @@ fi
 repo="$1"
 version_dotted="$2"
 
-METADATA_FNAME="__metadata__.py"
-MANIFEST_FNAME="manifest.txt"
+pushd $repo
 
-GENERAL_CONFIG_DIR="\/etc\/indy"
-REPO_GENERAL_CONFIG_DIR="$repo/indy_node/general_config"
-
-echo -e "\n\nAbout to start updating package $repo to version $version_dotted info from cur dir: $(pwd)"
+echo -e "\nSetting version to $version_dotted"
+bash -ex ./bump_version.sh $version_dotted
+cat indy_node/__version__.json
 
-metadata="$(find $repo -name $METADATA_FNAME)"
+echo -e "\nGenerating manifest"
+bash -ex ./generate_manifest.sh
+cat indy_node/__manifest__.json
 
-if [ -z $metadata ] ; then
-  echo "FAILED finding metadata"
-  exit $ret
-fi
+echo -e "\nAdapt the dependencies for the Canonical archive"
+sed -i "s~python-dateutil~python3-dateutil~" setup.py
+sed -i "s~timeout-decorator~python3-timeout-decorator~" setup.py
+sed -i "s~distro~python3-distro~" setup.py
 
-version=$(sed -r "s/\./, /g" <<< $version_dotted)
-
-echo -e "\n\nUpdating version in $metadata with $version"
-sed -i -r "s~(__version_info__ = \()[0-9, ]+~\1$version~" "$metadata"
-ret=$?
-if [ $ret -ne 0 ] ; then
-  echo "FAILED ret: $ret"
-  exit $ret
-fi
 
-echo -e "\n\nPrepares indy-plenum debian package version"
-sed -i -r "s~indy-plenum==([0-9\.]+[0-9])(\.)?([a-z]+)~indy-plenum==\1\~\3~" $repo/setup.py
+echo -e "\nPreparing config files"
 
-echo -e "Adapt the dependencies for the Canonical archive"
-sed -i "s~python-dateutil~python3-dateutil~" "$repo/setup.py"
-sed -i "s~timeout-decorator~python3-timeout-decorator~" "$repo/setup.py"
-sed -i "s~distro~python3-distro~" "$repo/setup.py"
-
-# create manifest file
-repourl=$(git --git-dir $repo/.git --work-tree $repo config --get remote.origin.url)
-hashcommit=$(git --git-dir $repo/.git --work-tree $repo rev-parse HEAD)
-manifest="// built from: repo version hash\n$repourl $version_dotted $hashcommit"
-manifest_file=$(echo $metadata | sed -r "s/${METADATA_FNAME}$/${MANIFEST_FNAME}/")
-
-echo "Adding manifest\n=======\n$manifest\n=======\n into $manifest_file"
-rm -rf $manifest_file
-echo -e $manifest >$manifest_file
+GENERAL_CONFIG_DIR="\/etc\/indy"
+REPO_GENERAL_CONFIG_DIR="indy_node/general_config"
 
-echo "Preparing config files"
 # Define user config directory
-sed -i "s/^\(GENERAL_CONFIG_DIR\s*=\s*\).*\$/\1\"$GENERAL_CONFIG_DIR\"/" "$repo/indy_common/config.py"
+sed -i "s/^\(GENERAL_CONFIG_DIR\s*=\s*\).*\$/\1\"$GENERAL_CONFIG_DIR\"/" indy_common/config.py
 # Create user config
 cp $REPO_GENERAL_CONFIG_DIR/general_config.py $REPO_GENERAL_CONFIG_DIR/indy_config.py
 cat $REPO_GENERAL_CONFIG_DIR/ubuntu_platform_config.py >> $REPO_GENERAL_CONFIG_DIR/indy_config.py
 rm -f $REPO_GENERAL_CONFIG_DIR/general_config.py
 rm -f $REPO_GENERAL_CONFIG_DIR/ubuntu_platform_config.py
 rm -f $REPO_GENERAL_CONFIG_DIR/windows_platform_config.py
 
-echo -e "Finished preparing $repo for publishing\n"
+popd
+
+echo -e "\nFinished preparing $repo for publishing\n"

bump_version.sh

@@ -1,10 +1,12 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e
 
 usage_str="Usage: $0 <version-dotted>"
 
 if [ -z "$1" ] ; then
     echo $usage_str
-    exit 0
+    exit 1
 fi
 
 if [ "$1" = "--help" ] ; then

generate_manifest.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+usage_str="Usage: $0"
+
+if [ "$1" = "--help" ] ; then
+  echo $usage_str
+  exit 0
+fi
+
+repourl=$(git config --get remote.origin.url)
+hashcommit=$(git $repo rev-parse HEAD)
+
+python3 -c "import indy_node; indy_node.set_manifest({'repo': '$repourl', 'sha1': '$hashcommit', 'version': indy_node.__version__})"

indy_node/__init__.py

@@ -1,7 +1,8 @@
 from .__metadata__ import (
-    __title__, __version_info__, __version__, __description__,
+    __title__, __version_info__, __version__, __manifest__, __description__,
     __long_description__, __keywords__, __url__, __author__,
-    __author_email__, __maintainer__, __license__, load_version, set_version
+    __author_email__, __maintainer__, __license__,
+    load_version, set_version, load_manifest, set_manifest
 )
 
 PLUGIN_LEDGER_IDS = set()

indy_node/__metadata__.py

@@ -3,7 +3,7 @@
 """
 import os
 import json
-from typing import Tuple, List, Union
+from typing import Any
 import collections.abc
 
 from indy_common.node_version import NodeVersion, InvalidVersionError
@@ -13,6 +13,11 @@
     os.path.abspath(os.path.dirname(__file__)), VERSION_FILENAME)
 
 
+MANIFEST_FILENAME = '__manifest__.json'
+MANIFEST_FILE = os.path.join(
+    os.path.abspath(os.path.dirname(__file__)), MANIFEST_FILENAME)
+
+
 # TODO use/wrap plenum's set and load API
 def load_version(version_file: str = VERSION_FILE) -> NodeVersion:
     with open(version_file, 'r') as _f:
@@ -32,13 +37,24 @@ def set_version(version: str, version_file: str = VERSION_FILE):
         _f.write('\n')
 
 
+def load_manifest(manifest_file: str = MANIFEST_FILE) -> Any:
+    try:
+        with open(manifest_file, 'r') as _f:
+            return json.load(_f)
+    except IOError as exc:
+        return None
+
+
+def set_manifest(manifest: Any, manifest_file: str = MANIFEST_FILE):
+    with open(manifest_file, 'w') as _f:
+        json.dump(manifest, _f)
+        _f.write('\n')
+
+
 __title__ = 'indy-node'
-__version_info__ = (1, 6)
-__version__ = '.'.join(map(str, __version_info__))
-# TODO activate once new versioning scheme becomes implemented
-# Note. double underscores
-# _version_info__ = load_version()
-# _version__ = __version_info__.full
+__version_info__ = load_version()
+__version__ = __version_info__.full
+__manifest__ = load_manifest()
 __description__ = 'Indy node'
 __long_description__ = __description__
 __keywords__ = 'Indy Node'
@@ -50,7 +66,7 @@ def set_version(version: str, version_file: str = VERSION_FILE):
 
 
 __all__ = [
-    '__title__', '__version_info__', '__version__',
+    '__title__', '__version_info__', '__version__', '__manifest__',
     '__description__', '__long_description__', '__keywords__',
     '__url__', '__author__', '__author_email__', '__maintainer__',
-    '__license__', 'load_version', 'set_version']
+    '__license__', 'load_version', 'set_version', 'load_manifest', 'set_manifest']

indy_node/test/package/test_metadata.py

@@ -3,17 +3,29 @@
 import os
 
 from indy_common.node_version import NodeVersion, InvalidVersionError
-from indy_node.__metadata__ import set_version, load_version
+from plenum.__metadata__ import (
+    set_version, load_version,
+    set_manifest, load_manifest
+)
 
 
-@pytest.fixture
-def version_file_path(tdir, request):
+def tmp_file_path(tdir, request, file_name):
     return os.path.join(
         tdir,
-        "{}.upgrade_log".format(os.path.basename(request.node.nodeid))
+        "{}.{}".format(os.path.basename(request.node.nodeid), file_name)
     )
 
 
+@pytest.fixture
+def version_file_path(tdir, request):
+    return tmp_file_path(tdir, request, '__version__.json')
+
+
+@pytest.fixture
+def manifest_file_path(tdir, request):
+    return tmp_file_path(tdir, request, '__manifest__.json')
+
+
 def idfn(v):
     return str(v).replace(' ', '')
 
@@ -71,3 +83,24 @@ def test_set_version_invalid(version, version_file_path):
 def test_set_load_version_valid(version, version_file_path):
     set_version(version, version_file_path)
     assert load_version(version_file_path) == NodeVersion(version)
+
+
+
+def test_load_absent_manifest(manifest_file_path):
+    assert load_manifest(manifest_file_path) is None
+
+
+# TODO ??? wider coverage
+@pytest.mark.parametrize(
+    'manifest',
+    [
+        1,
+        [1, 2],
+        None,
+        {'k1': 'v1', "2": 2},
+    ],
+    ids=idfn
+)
+def test_set_load_manifest_valid(manifest, manifest_file_path):
+    set_manifest(manifest, manifest_file_path)
+    assert load_manifest(manifest_file_path) == manifest

setup.py

@@ -42,6 +42,7 @@
     license=metadata['__license__'],
     packages=find_packages(exclude=['docs', 'docs*']) + [
         'data'],
+    # TODO move that to MANIFEST.in
     package_data={
         '': ['*.txt', '*.md', '*.rst', '*.json', '*.conf', '*.html',
              '*.css', '*.ico', '*.png', 'LICENSE', 'LEGAL', '*.indy']},