Github Actions (trailofbits#1681)

sashimi0 · Jan 13, 2020 · 53dfc57 · 53dfc57
1 parent eb40ade
commit 53dfc57
Show file tree

Hide file tree

Showing 8 changed files with 177 additions and 143 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -0,0 +1,160 @@
+name: Main
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+
+      - name: Install dependencies
+        run: |
+          sudo apt update -y
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          sudo snap install shellcheck
+          pip install ansible-lint
+
+      - name: Checks and linters
+        run: |
+          /snap/bin/shellcheck algo install.sh
+          ansible-playbook main.yml --syntax-check
+          ansible-lint -v *.yml roles/{local,cloud-*}/*/*.yml
+
+  scripted-deploy:
+    runs-on: ubuntu-16.04
+    strategy:
+      matrix:
+        UBUNTU_VERSION: ["18.04", "19.04", "19.10"]
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+
+      - name: Install dependencies
+        run: |
+          sudo apt update -y
+          sudo add-apt-repository -yu ppa:wireguard/wireguard
+          sudo apt install -y \
+            python3-pip \
+            lxd \
+            expect-dev \
+            debootstrap \
+            tree \
+            bridge-utils \
+            dnsutils \
+            build-essential \
+            libssl-dev \
+            libffi-dev \
+            python3-dev \
+            linux-headers-$(uname -r) \
+            wireguard \
+            libxml2-utils \
+            crudini \
+            fping \
+            strongswan \
+            libstrongswan-standard-plugins \
+            resolvconf
+
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements.txt
+
+      - name: Provision
+        env:
+          DEPLOY: cloud-init
+          UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }}
+        run: |
+          ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
+          # sed -i "s/^reduce_mtu:\s0$/reduce_mtu: 80/" config.cfg
+          sudo -E ./tests/pre-deploy.sh
+
+      - name: Deployment
+        run: |
+          until sudo lxc exec algo -- test -f /var/log/cloud-init-output.log; do echo 'Log file not found, Sleep for 3 seconds'; sleep 3; done
+          ( sudo lxc exec algo -- tail -f /var/log/cloud-init-output.log & )
+          until sudo lxc exec algo -- test -f /var/lib/cloud/data/result.json; do
+            echo 'Cloud init is not finished. Sleep for 30 seconds';
+            sleep 30;
+          done
+          sudo lxc exec algo -- test -f /opt/algo/configs/localhost/.config.yml
+          sudo lxc exec algo -- tar zcf /root/algo-configs.tar -C /opt/algo/configs/ .
+          sudo lxc file pull algo/root/algo-configs.tar ./
+          sudo tar -C ./configs -zxf algo-configs.tar
+
+      - name: Tests
+        run: |
+          set -x
+          sudo -E bash -x ./tests/wireguard-client.sh
+          sudo env "PATH=$PATH" ./tests/ipsec-client.sh
+          sudo ./tests/ssh-tunnel.sh
+
+  local-deploy:
+    runs-on: ubuntu-16.04
+    strategy:
+      matrix:
+        UBUNTU_VERSION: ["18.04", "19.04", "19.10"]
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+
+      - name: Install dependencies
+        run: |
+          set -x
+          sudo add-apt-repository -yu ppa:wireguard/wireguard
+          sudo add-apt-repository -yu ppa:ubuntu-lxc/stable
+          sudo apt update -y
+          sudo apt install -y \
+            python3-pip \
+            lxd \
+            expect-dev \
+            debootstrap \
+            tree \
+            bridge-utils \
+            dnsutils \
+            build-essential \
+            libssl-dev \
+            libffi-dev \
+            python3-dev \
+            linux-headers-$(uname -r) \
+            wireguard \
+            libxml2-utils \
+            crudini \
+            fping \
+            strongswan \
+            libstrongswan-standard-plugins \
+            resolvconf
+
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements.txt
+
+      - name: Provision
+        env:
+          DEPLOY: docker
+          UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }}
+        run: |
+          ssh-keygen -f ~/.ssh/id_rsa -t rsa -N ''
+          sed -i "s/^reduce_mtu:\s0$/reduce_mtu: 80/" config.cfg
+          sudo -E ./tests/pre-deploy.sh
+
+      - name: Deployment
+        env:
+          DEPLOY: docker
+          UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }}
+        run: |
+          docker build -t local/algo .
+          ./tests/local-deploy.sh
+          ./tests/update-users.sh
+
+      - name: Tests
+        run: |
+          set -x
+          sudo bash -x ./tests/wireguard-client.sh
+          sudo env "PATH=$PATH" bash -x ./tests/ipsec-client.sh
+          sudo bash -x ./tests/ssh-tunnel.sh
diff --git a/.travis.yml b/.travis.yml
diff --git a/server.yml b/server.yml
@@ -87,8 +87,8 @@
               ipsec_enabled: {{ ipsec_enabled }}
               wireguard_enabled: {{ wireguard_enabled }}
               {% if tests|default(false)|bool %}
-              ca_password: {{ CA_password }}
-              p12_password: {{ p12_export_password }}
+              ca_password: '{{ CA_password }}'
+              p12_password: '{{ p12_export_password }}'
               {% endif %}
           become: false
           delegate_to: localhost

diff --git a/tests/cloud-init.sh b/tests/cloud-init.sh
@@ -9,9 +9,9 @@ export DNS_ADBLOCKING=true
 export SSH_TUNNELING=true
 export ENDPOINT=10.0.8.100
 export USERS=desktop,user1,user2
-export EXTRA_VARS='install_headers=false tests=true apparmor_enabled=false local_service_ip=172.16.0.1'
-export ANSIBLE_EXTRA_ARGS='--skip-tags apparmor'
-export REPO_SLUG=${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG:-trailofbits/algo}}
-export REPO_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH:-master}}
+export EXTRA_VARS='install_headers=false tests=true local_service_ip=172.16.0.1'
+export ANSIBLE_EXTRA_ARGS=''
+export REPO_SLUG=${REPOSITORY:-trailofbits/algo}
+export REPO_BRANCH=${BRANCH:-master}
 
-curl -s https://raw.githubusercontent.com/${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG}}/${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}/install.sh | sudo -E bash -x"
+curl -s https://raw.githubusercontent.com/${REPOSITORY:-trailofbits/algo}/${BRANCH:-master}/install.sh | sudo -E bash -x"
diff --git a/tests/local-deploy.sh b/tests/local-deploy.sh
@@ -2,11 +2,11 @@
 
 set -ex
 
-DEPLOY_ARGS="provider=local server=10.0.8.100 ssh_user=ubuntu endpoint=10.0.8.100 apparmor_enabled=false ondemand_cellular=true ondemand_wifi=true ondemand_wifi_exclude=test dns_adblocking=true ssh_tunneling=true store_pki=true install_headers=false tests=true local_service_ip=172.16.0.1"
+DEPLOY_ARGS="provider=local server=10.0.8.100 ssh_user=ubuntu endpoint=10.0.8.100 ondemand_cellular=true ondemand_wifi=true ondemand_wifi_exclude=test dns_adblocking=true ssh_tunneling=true store_pki=true install_headers=false tests=true local_service_ip=172.16.0.1"
 
 if [ "${DEPLOY}" == "docker" ]
 then
-  docker run -it -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "DEPLOY_ARGS=${DEPLOY_ARGS}" travis/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook main.yml -e \"${DEPLOY_ARGS}\" --skip-tags apparmor"
+  docker run -i -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "DEPLOY_ARGS=${DEPLOY_ARGS}" local/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook main.yml -e \"${DEPLOY_ARGS}\" --skip-tags debug"
 else
-  ansible-playbook main.yml -e "${DEPLOY_ARGS}" --skip-tags apparmor
+  ansible-playbook main.yml -e "${DEPLOY_ARGS}"
 fi
diff --git a/tests/pre-deploy.sh b/tests/pre-deploy.sh
@@ -8,6 +8,9 @@ tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache."
 cp -f tests/lxd-bridge /etc/default/lxd-bridge
 cp -f tests/algo.conf /etc/default/algo.conf
 
+export REPOSITORY=${GITHUB_REPOSITORY}
+export BRANCH=${GITHUB_REF#refs/heads/}
+
 if [[ "$DEPLOY" == "cloud-init" ]]; then
   bash tests/cloud-init.sh | lxc profile set default user.user-data -
 else

diff --git a/tests/ssh-tunnel.sh b/tests/ssh-tunnel.sh
@@ -2,14 +2,14 @@
 
 set -euxo pipefail
 
-PASS=$(grep ^p12_password: configs/10.0.8.100/.config.yml | awk '{print $2}')
+PASS=$(grep ^p12_password: configs/10.0.8.100/.config.yml | awk '{print $2}' | cut -f2 -d\')
 
 ssh-keygen -p -P ${PASS} -N '' -f configs/10.0.8.100/ssh-tunnel/desktop.pem
 
 ssh -o StrictHostKeyChecking=no -D 127.0.0.1:1080 -f -q -C -N [email protected] -i configs/10.0.8.100/ssh-tunnel/desktop.pem -F configs/10.0.8.100/ssh_config
 
 git config --global http.proxy 'socks5://127.0.0.1:1080'
 
-git clone -vv https://github.com/trailofbits/algo /tmp/ssh-tunnel-check
+for i in {1..10}; do git clone -vv https://github.com/trailofbits/algo /tmp/ssh-tunnel-check && break || sleep 1; done
 
 echo "SSH tunneling tests passed"
diff --git a/tests/update-users.sh b/tests/update-users.sh
@@ -6,7 +6,7 @@ USER_ARGS="{ 'server': '10.0.8.100', 'users': ['desktop', 'user1', 'user2'], 'lo
 
 if [ "${DEPLOY}" == "docker" ]
 then
-  docker run -it -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "USER_ARGS=${USER_ARGS}" travis/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook users.yml -e \"${USER_ARGS}\" -t update-users"
+  docker run -i -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "USER_ARGS=${USER_ARGS}" local/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook users.yml -e \"${USER_ARGS}\" -t update-users --skip-tags debug -vvvvv"
 else
   ansible-playbook users.yml -e "${USER_ARGS}" -t update-users
 fi
@@ -23,7 +23,7 @@ if sudo openssl crl -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/c
     exit 1
 fi
 
-if sudo openssl x509 -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/certs/user1.crt | grep CN=user1
+if sudo openssl x509 -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/certs/user1.crt | grep -E "CN(=|\s+=\s+)user1"
   then
     echo "The new user exists"
   else