More improvements to CircleCI configuration (getredash#2777)

* Moved configuration to `.circleci/config.yml` as the documentation for V2 suggests.
* Created a dedicated Docker Compose configuration (in `.circleci/docker-compose.yml`) to remove volumes configuration as this is not supported with CircleCI's Docker executer.
* Fix the Docker image build and tarball packing jobs to work and use correct version.

circle.yml → .circleci/config.yml

@@ -1,65 +1,64 @@
 version: 2.0
 jobs:
-  npm-build:
+  unit-tests:
+    environment:
+      COMPOSE_FILE: .circleci/docker-compose.circle.yml
+      COMPOSE_PROJECT_NAME: redash
     docker:
-      - image: circleci/node:8
-    working_directory: ~/redash
+      - image: circleci/buildpack-deps:xenial
     steps:
+      - setup_remote_docker
       - checkout
-      - restore_cache:
-         keys:
-           - node-modules-
-      - run:
-          name: Setup NPM Environment
-          command: |
-            npm install
-      - run:
-          name: build
-          command: npm run build
-      - persist_to_workspace:
-          root: .
-          paths:
-            - ./*
-      - save_cache:
-         key: node-modules-
-         paths:
-           - node_modules
-  unit-tests:
-    machine: true
-    working_directory: ~/redash
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: Install Docker Compose
-          command: |
-            set -x
-            pip install --upgrade pip
-            pip install docker-compose>=1.18
-            docker-compose --version
       - run:
-          name: Pull images and setup
+          name: Build Docker Images
           command: |
             set -x
             docker-compose up -d
             sleep 10
-            docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests;"
       - run:
-          name: Run tests
-          command: docker-compose run --user root server tests
-      - store_artifacts:
-          path: junit.xml
+          name: Create Test Database
+          command: docker-compose run --rm postgres psql -h postgres -U postgres -c "create database tests;"
+      - run:
+          name: Run Tests
+          command: docker-compose run --name tests redash tests --junitxml=junit.xml tests/
+      - run:
+          name: Copy Test Results
+          command: |
+            mkdir -p /tmp/test-results/unit-tests
+            docker cp tests:/app/coverage.xml ./coverage.xml 
+            docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
+      - store_test_results:
+          path: /tmp/test-results
       - store_artifacts:
           path: coverage.xml
+  build-tarball:
+    docker:
+      - image: circleci/node:8
+    steps:
+      - checkout
+      - run: npm install
+      - run: npm run build
+      - run: .circleci/update_version
+      - run: .circleci/pack
+      - store_artifacts:
+          path: /tmp/artifacts/
+  build-docker-image:
+    docker:
+      - image: circleci/buildpack-deps:xenial
+    steps:
+      - checkout
+      - run: .circleci/update_version
+      - run: docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+      - run: docker build -t redash/redash:$(.circleci/docker_tag) .
+      - run: docker push redash/redash:$(.circleci/docker_tag)
   integration-tests:
     working_directory: ~/redash
     machine: true
     environment:
       REDASH_SERVER_URL : "http://127.0.0.1:5000/"
       DOCKER_IMAGE: mozilla/redash-ui-tests
     steps:
-      - attach_workspace:
-          at: .
+      - checkout
       - run:
           name: Install Docker Compose
           command: |
@@ -93,56 +92,23 @@ jobs:
             docker run --net="host" --env REDASH_SERVER_URL="${REDASH_SERVER_URL}" "${DOCKER_IMAGE}"
       - store_artifacts:
           path: report.html
-  build-tarball: # build for master or release branches
-    docker:
-      - image: circleci/node:8
-    environment:
-      NAME: redash
-      VERSION: $(python ./manage.py version)
-      FULL_VERSION: $VERSION+b$CIRCLE_BUILD_NUM
-      FILENAME: $NAME.$FULL_VERSION.tar.gz
-    working_directory: ~/redash
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: Pack Tarball
-          command: bin/pack "$CIRCLE_BRANCH"
-      - store_artifacts:
-          path: $NAME.$FULL_VERSION.tar.gz
-  build-docker-image: # build for tags
-    docker:
-      - image: circleci/node:8
-    working_directory: ~/redash
-    steps:
-      - attach_workspace:
-          at: .
-      - run:
-          name: Tag deploy
-          command: |
-            set -x
-            docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
-            docker build -t redash/redash:$(./manage.py version | sed -e "s/\+/./") .
-            docker push redash/redash:$(./manage.py version | sed -e "s/\+/./")
 workflows:
   version: 2
   integration_tests:
     jobs:
       - integration-tests:
           filters:
              branches:
-               only:
-                 - master
+               only: master
   build:
     jobs:
-      - npm-build
-      - unit-tests:
-          requires:
-            - npm-build
+      - unit-tests
       - build-tarball:
            requires:
              - unit-tests
            filters:
+             tags:
+               only: /v[0-9]+(\.[0-9\-a-z]+)*/
              branches:
                only:
                  - master
@@ -152,6 +118,4 @@ workflows:
              - unit-tests
            filters:
              tags:
-               only: /v[0-9]+(\.[0-9\-a-z]+)*/
-             branches:
-               ignore: /.*/
+               only: /v[0-9]+(\.[0-9\-a-z]+)*/

.circleci/docker-compose.circle.yml

@@ -0,0 +1,22 @@
+version: '2'
+services:
+  redash:
+    build: ../
+    command: manage version
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+  redis:
+    image: redis:3.0-alpine
+    restart: unless-stopped
+  postgres:
+    image: postgres:9.5.6-alpine
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"
+    restart: unless-stopped

.circleci/docker_tag

@@ -0,0 +1,5 @@
+#!/bin/bash
+VERSION=$(jq -r .version package.json)
+FULL_VERSION=$VERSION.b$CIRCLE_BUILD_NUM
+
+echo $FULL_VERSION

.circleci/pack

@@ -0,0 +1,9 @@
+#!/bin/bash
+NAME=redash
+VERSION=$(jq -r .version package.json)
+FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM
+FILENAME=$NAME.$FULL_VERSION.tar.gz
+
+mkdir -p /tmp/artifacts/
+
+tar -zcv -f /tmp/artifacts/$FILENAME --exclude="optipng*" --exclude=".git*" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" --exclude="node_modules" *

.circleci/update_version

@@ -0,0 +1,5 @@
+#!/bin/bash
+VERSION=$(jq -r .version package.json)
+FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM
+
+sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '$FULL_VERSION'/" redash/__init__.py

.github/config.yml

@@ -0,0 +1,5 @@
+# https://github.com/behaviorbot/request-info?installation_id=189571
+requestInfoLabelToAdd: needs-more-info
+requestInfoReplyComment: >
+  We would appreciate it if you could provide us with more info about this issue/pr!
+

bin/get_changes.py

@@ -0,0 +1,35 @@
+#!/bin/env python
+import sys
+import re
+import subprocess
+
+def get_change_log(previous_sha):
+    args = ['git', '--no-pager', 'log', '--merges', '--grep', 'Merge pull request', '--pretty=format:"%h|%s|%b|%p"', 'master...{}'.format(previous_sha)]
+    log = subprocess.check_output(args)
+    changes = []
+
+    for line in log.split('\n'):
+        try:
+            sha, subject, body, parents = line[1:-1].split('|')
+        except ValueError:
+            continue
+
+        try:
+            pull_request = re.match("Merge pull request #(\d+)", subject).groups()[0]
+            pull_request = " #{}".format(pull_request)
+        except Exception as ex:
+            pull_request = ""
+
+        author = subprocess.check_output(['git', 'log', '-1', '--pretty=format:"%an"', parents.split(' ')[-1]])[1:-1]
+
+        changes.append("{}{}: {} ({})".format(sha, pull_request, body.strip(), author))
+
+    return changes
+
+
+if __name__ == '__main__':
+    previous_sha = sys.argv[1]
+    changes = get_change_log(previous_sha)
+
+    for change in changes:
+        print change