Github Actions (openethereum#11528)

* actions

* add build scripts

* chmod +x

* remove clang env

* add ARM builds

* install LLVM for Windows

* check LLVM

* remove fi

* check platform

* fix LLVM install on windows

* diff cache

* remove unexpected token &&

* remove build directory cache from windows

* remove Show ENV

* remove checksum

* fix cross and RUSTFLAGS

* final fix for ARM and Windows build

* typo

* fix PATH for artifacts

* sudo for install
set PATH . for artifacts.zip

* target names for artifacts

* platform names for artifacts

* remove ARM builds
PLATFORM for build scripts

* remove PLATFORM and change PATH for artifacts.zip

* change PATH for artifacts

* add sccache
div to test and build

* env RUSTC_WRAPPER sccache

* sccache tag 0.2.13 
up cache to latest v1.1.2
set SCCACHE_CACHE_SIZE: "1G"

* remove dependencies from build

* add audit check
add tags for Build sccache --show-stats

* remove beta and nightly toolchains

* refactoring cache && sccache

* fix Windows PATH o sccache HOME

* "*"->'*'

* install scache from https://github.com/mozilla/sccache/releases

* fix lint

* url for sccache

* sccache up

* shell: pwsh for sccache

* up install script for sccache

* fix link

* splitting tests into multiple tasks

* Jobs renaming

* typo fix

* remove Continue on error

* TEST: remove cargo build cache

* TEST: sccache show stats before

* TEST: 1

* TEST: 2

* TEST: 3 
MacOS fix sccache PATH

* TEST 4: enable target cache

* remove build

* TEST 5: check cache and up Cargo.toml

* TEST 6

* TEST 7: restore cargo cache
splt build-tests and run-tests

* TEST 8: fix build-test

* TEST 9: fix sccache

* TEST 10: lint run-test

* TEST 11: build and test in one job

* prepare for merge

* test.yml -> check.yml

* add install sccache script

* sh -> ps1

* add os arg to ps cript

* os up

* fix os arg

* ls sccache tar

* echo url

* echo sccache url

* add swith for select platform

* fix args

* TEST CLEAN 1: clean biuld dir for all platforms

* TEST CLEAN 2: incude build step 
continue-on-error in test

* TEST CLEAN 3: script for clean

* ready for merge

.github/workflows/audit.yml

@@ -0,0 +1,18 @@
+name: Security audit
+on:
+  pull_request:
+    paths: Cargo.lock
+  schedule:
+    - cron: '0 0 * * *'
+jobs:
+  security_audit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 50
+      - name: Run cargo audit
+        uses: actions-rs/audit-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/build-test.yml

@@ -0,0 +1,98 @@
+name:                           Build and Test Suite
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - stable
+jobs:
+  build-tests:
+    name:                       Test and Build
+    strategy:
+      matrix:
+        platform:
+          - ubuntu-16.04
+          - macos-latest
+          - windows-latest
+        toolchain:
+          - stable
+    runs-on:                    ${{ matrix.platform }}
+    steps:
+      - name:                   Checkout sources
+        uses:                   actions/checkout@v2
+        with:
+          fetch-depth:          50
+      - name:                   Checkout submodules
+        shell:                  bash
+        run:                    git submodule update --init --recursive
+      - name:                   Install toolchain
+        uses:                   actions-rs/toolchain@v1
+        with:
+          toolchain:            ${{ matrix.toolchain }}
+          override:             true
+      - name:                   Cache cargo registry
+        uses:                   actions/[email protected]
+        with:
+          path:                 ~/.cargo/registry
+          key:                  ${{ runner.os }}-cargo-registry-build-tests-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache cargo index
+        uses:                   actions/[email protected]
+        with:
+          path:                 ~/.cargo/git
+          key:                  ${{ runner.os }}-cargo-git-build-tests-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache cargo build
+        uses:                   actions/[email protected]
+        with:
+          path:                 target
+          key:                  ${{ runner.os }}-cargo-build-target-build-tests-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache sccache linux
+        if:                     matrix.platform == 'ubuntu-16.04'
+        uses:                   actions/[email protected]
+        with:
+          path:                 "/home/runner/.cache/sccache"
+          key:                  ${{ runner.os }}-sccache-build-tests-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache sccache MacOS
+        if:                     matrix.platform == 'macos-latest'
+        uses:                   actions/[email protected]
+        with:
+          path:                 "/Users/runner/Library/Caches/Mozilla.sccache"
+          key:                  ${{ runner.os }}-sccache-build-tests-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache sccache Windows
+        if:                     matrix.platform == 'windows-latest'
+        uses:                   actions/[email protected]
+        with:
+          path:                 "C:\\Users\\runneradmin\\AppData\\Local\\Mozilla\\sccache\\cache"
+          key:                  ${{ runner.os }}-sccache-build-tests-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Install sccache for ${{ matrix.platform }}
+        shell:                  pwsh
+        run:                    pwsh scripts/actions/install-sccache.ps1 ${{ runner.os}}
+      - name:                   Install LLVM for Windows
+        if:                     matrix.platform == 'windows-latest'
+        run:                    choco install llvm
+      - name:                   Sccache statistics
+        run:                    sccache --show-stats
+      - name:                   Build tests
+        uses:                   actions-rs/cargo@v1
+        with:
+          command:              test
+          args:                 --locked --all --release --features "json-tests" --verbose --no-run
+      - name:                   Run tests for ${{ matrix.platform }}
+        if:                     matrix.platform == 'windows-latest'
+        continue-on-error: true #Skip step if Windows tests failure
+        uses:                   actions-rs/cargo@v1
+        with:
+          command:              test
+          args:                 --locked --all --release --features "json-tests" --verbose
+      - name:                   Run tests for ${{ matrix.platform }}
+        if:                     matrix.platform != 'windows-latest'
+        uses:                   actions-rs/cargo@v1
+        with:
+          command:              test
+          args:                 --locked --all --release --features "json-tests" --verbose
+      - name:                   Stop sccache
+        if:                     always()
+        run:                    sccache --stop-server
+      - name:                   Prepare build directory for cache
+        shell: bash
+        run:                    bash scripts/actions/clean-target.sh

.github/workflows/build.yml

@@ -0,0 +1,90 @@
+name:                           Build Release Suite
+
+on:
+  push:
+    branches:
+      - stable
+    tags:
+      - v2*
+
+jobs:
+  build:
+    name:                       Build Release
+    strategy:
+      matrix:
+        platform:
+          - ubuntu-16.04
+          - macos-latest
+          - windows-latest
+        toolchain:
+          - stable
+    runs-on:                    ${{ matrix.platform }}
+    steps:
+      - name:                   Checkout sources
+        uses:                   actions/checkout@v2
+        with:
+          fetch-depth:          50
+      - name:                   Install toolchain
+        uses:                   actions-rs/toolchain@v1
+        with:
+          toolchain:            ${{ matrix.toolchain }}
+          profile:              minimal
+          override:             true
+      - name:                   Cache cargo registry
+        uses:                   actions/[email protected]
+        with:
+          path:                 ~/.cargo/registry
+          key:                  ${{ runner.os }}-cargo-registry-build-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache cargo index
+        uses:                   actions/[email protected]
+        with:
+          path:                 ~/.cargo/git
+          key:                  ${{ runner.os }}-cargo-git-build-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache cargo build
+        uses:                   actions/[email protected]
+        with:
+          path:                 target
+          key:                  ${{ runner.os }}-cargo-build-target-build-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache sccache linux
+        if:                     matrix.platform == 'ubuntu-16.04'
+        uses:                   actions/[email protected]
+        with:
+          path:                 "/home/runner/.cache/sccache"
+          key:                  ${{ runner.os }}-sccache-build-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache sccache MacOS
+        if:                     matrix.platform == 'macos-latest'
+        uses:                   actions/[email protected]
+        with:
+          path:                 "/Users/runner/Library/Caches/Mozilla.sccache"
+          key:                  ${{ runner.os }}-sccache-build-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache sccache Windows
+        if:                     matrix.platform == 'windows-latest'
+        uses:                   actions/[email protected]
+        with:
+          path:                 "C:\\Users\\runneradmin\\AppData\\Local\\Mozilla\\sccache\\cache"
+          key:                  ${{ runner.os }}-sccache-build-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Install sccache for ${{ matrix.platform }}
+        shell:                  pwsh
+        run:                    pwsh scripts/actions/install-sccache.ps1 ${{ runner.os}}
+      - name:                   Install LLVM for Windows
+        if:                     matrix.platform == 'windows-latest'
+        run:                    choco install llvm
+      - name:                   Sccache statistics
+        run:                    sccache --show-stats
+      - name:                   Build OpenEthereum for windows
+        if:                     matrix.platform == 'windows-latest'
+        run:                    sh scripts/actions/build-windows.sh ${{matrix.platform}}
+      - name:                   Build OpenEthereum for ${{matrix.platform}}
+        if:                     matrix.platform != 'windows-latest'
+        run:                    sh scripts/actions/build-linux.sh ${{matrix.platform}}
+      - name:                   Stop sccache
+        if:                     always()
+        run:                    sccache --stop-server
+      - name:                   Download artifact
+        uses:                   actions/upload-artifact@v1
+        with:
+          name:                 ${{matrix.platform}}.artifacts.zip
+          path:                 artifacts/
+      - name:                   Prepare build directory for cache
+        shell: bash
+        run:                    bash scripts/actions/clean-target.sh

.github/workflows/check.yml

@@ -0,0 +1,85 @@
+name:                           Check
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - stable
+jobs:
+  check:
+    name:                       Check
+    runs-on:                    ubuntu-16.04
+    steps:
+      - name:                   Checkout sources
+        uses:                   actions/checkout@v2
+        with:
+          fetch-depth:          50
+      - name:                   Checkout submodules
+        shell:                  bash
+        run:                    git submodule update --init --recursive
+      - name:                   Install stable toolchain
+        uses:                   actions-rs/toolchain@v1
+        with:
+          toolchain:            stable
+          profile:              minimal
+          override:             true
+      - name:                   Cache cargo registry
+        uses:                   actions/[email protected]
+        with:
+          path:                 ~/.cargo/registry
+          key:                  ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache cargo index
+        uses:                   actions/[email protected]
+        with:
+          path:                 ~/.cargo/git
+          key:                  ${{ runner.os }}-cargo-git-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Cache cargo build
+        uses:                   actions/[email protected]
+        with:
+          path:                 target
+          key:                  ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+      # Install sccache based on https://github.com/denoland/rusty_v8/blob/master/.github/workflows/ci.yml#L69
+      - name:                   Cache sccache
+        uses:                   actions/[email protected]
+        with:
+          path:                 "/home/runner/.cache/sccache"
+          key:                  ${{ runner.os }}-sccache-check-${{ hashFiles('**/Cargo.lock') }}
+      - name:                   Install sccache for Linux
+        shell:                  pwsh
+        run:                    pwsh scripts/actions/install-sccache.ps1 ${{ runner.os}}
+      - name:                   Sccache statistics
+        run:                    sccache --show-stats
+      - name:                   Run cargo check 1/3
+        uses:                   actions-rs/cargo@v1
+        with:
+          command:              check
+          args:                 --locked --no-default-features --verbose
+      - name:                   Run cargo check 2/3
+        uses:                   actions-rs/cargo@v1
+        with:
+          command:              check
+          args:                 --locked --manifest-path util/io/Cargo.toml --no-default-features --verbose
+      - name:                   Run cargo check 3/3
+        uses:                   actions-rs/cargo@v1
+        with:
+          command:              check
+          args:                 --locked --manifest-path util/io/Cargo.toml --features "mio" --verbose
+      - name:                   Run cargo check evmbin
+        uses:                   actions-rs/cargo@v1
+        with:
+          command:              check
+          args:                 --locked -p evmbin --verbose
+      - name:                   Run cargo check benches
+        uses:                   actions-rs/cargo@v1
+        with:
+          command:              check
+          args:                 --locked --all --benches --verbose
+      - name:                   Run validate chainspecs
+        run:                    ./scripts/actions/validate-chainspecs.sh
+      - name:                   Stop sccache
+        if:                     always()
+        run:                    sccache --stop-server
+      - name:                   Prepare build directory for cache
+        shell: bash
+        run:                    bash scripts/actions/clean-target.sh

scripts/actions/build-linux.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+#strip ON
+export RUSTFLAGS=" -Clink-arg=-s -Ctarget-feature=+aes,+sse2,+ssse3"
+
+echo "_____ Build OpenEthereum and tools _____"
+
+time cargo build --verbose --color=always --release --features final
+time cargo build --verbose --color=always --release -p evmbin
+time cargo build --verbose --color=always --release -p ethstore-cli
+time cargo build --verbose --color=always --release -p ethkey-cli
+
+echo "_____ Post-processing binaries _____"
+rm -rf artifacts/*
+mkdir -p artifacts/
+
+cp -v target/release/parity artifacts/parity
+cp -v target/release/parity-evm artifacts/parity-evm
+cp -v target/release/ethstore artifacts/ethstore
+cp -v target/release/ethkey artifacts/ethkey

scripts/actions/build-windows.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e # fail on any error
+set -u # treat unset variables as error
+
+  # NOTE: Enables the aes-ni instructions for RustCrypto dependency.
+  # If you change this please remember to also update .cargo/config
+export RUSTFLAGS=" -Ctarget-feature=+aes,+sse2,+ssse3 -Ctarget-feature=+crt-static  -Clink-arg=-s"
+
+echo "_____ Build OpenEthereum and tools _____"
+time cargo build --verbose --release --features final
+time cargo build --verbose --release -p evmbin
+time cargo build --verbose --release -p ethstore-cli
+time cargo build --verbose --release -p ethkey-cli
+
+echo "_____ Post-processing binaries _____"
+rm -rf artifacts
+mkdir -p artifacts
+
+cp --verbose target/release/parity.exe artifacts/parity.exe
+cp --verbose target/release/parity-evm.exe artifacts/parity-evm.exe
+cp --verbose target/release/ethstore.exe artifacts/ethstore.exe
+cp --verbose target/release/ethkey.exe artifacts/ethkey.exe

scripts/actions/clean-target.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e # fail on any error
+set -u # treat unset variables as error
+
+find ./target/release -maxdepth 1 -type f -delete;
+rm -fr ./target/release/{deps,.fingerprint}/*{parity,ethcore,ethkey,ethstore,parity-evm}*;
+rm -f  ./target/.rustc_info.json;

scripts/actions/install-sccache.ps1

@@ -0,0 +1,21 @@
+#!/usr/bin/env pwsh
+$os=$args[0]
+$SCCACHE_CACHE_SIZE="1G"
+$SCCACHE_IDLE_TIMEOUT=0
+$version="0.2.12"
+echo "Current OS:" $os
+switch ($os){
+   "macOS" {$platform = "x86_64-apple-darwin"}
+   "Linux" {$platform = "x86_64-unknown-linux-musl"}
+   "Windows"  {$platform ="x86_64-pc-windows-msvc"}
+}
+echo "Target arch: " $platform
+$basename = "sccache-$version-$platform"
+$url = "https://github.com/mozilla/sccache/releases/download/"+"$version/$basename.tar.gz"
+echo "Download sccache from "+$url
+curl -LO $url
+tar -xzvf "$basename.tar.gz"
+ls $basename/
+. $basename/sccache --start-server
+echo "::add-path::$(pwd)/$basename"
+echo "::set-env name=RUSTC_WRAPPER::sccache"

scripts/actions/validate-chainspecs.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e # fail on any error
+set -u # treat unset variables as error
+echo "________Running validate_chainspecs.sh________"
+
+ERR=0
+
+echo "________Validate chainspecs________"
+time cargo build --release -p chainspec --verbose --color=always
+
+for spec in ethcore/res/*.json; do
+    if ! ./target/release/chainspec "$spec"; then ERR=1; fi
+done
+
+for spec in ethcore/res/ethereum/*.json; do
+    if ! ./target/release/chainspec "$spec"; then ERR=1; fi
+done
+
+echo "________Mainnet contains Istanbul EIPs________"
+for eip in $(grep --only-matching "eip.*Transition" ethcore/res/ethereum/istanbul_test.json); do
+    if ! grep -q $eip ethcore/res/ethereum/foundation.json; then
+        echo "ERROR: $eip is missing in the foundation json spec"
+        ERR=1
+    fi
+done
+
+exit $ERR