Setup script for Ubuntu/Debian + packer configuration

This script is intended to work on Ubuntu 12.04, Ubuntu 14.04 and Debian Wheezy (for GCE users). To make sure we use the same version of Redis across all distributions we install from source, and to make sure we use the same version of PostgreSQL we install it from PostgreSQL's apt. Also included Packer configuration to generate GCE & AWS images.
nkabir · Oct 21, 2014 · f523378 · f523378
1 parent 4720e12
commit f523378
Show file tree

Hide file tree

Showing 10 changed files with 1,432 additions and 0 deletions.
diff --git a/setup/Vagrantfile_debian b/setup/Vagrantfile_debian
@@ -0,0 +1,12 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  # Every Vagrant virtual environment requires a box to build off of.
+  config.vm.box = "box-cutter/debian76"
+  config.vm.provision "shell", path: "setup.sh"
+  config.vm.network "forwarded_port", guest: 80, host: 9001
+end
diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh
@@ -0,0 +1,172 @@
+#!/bin/bash
+set -eu
+
+REDASH_BASE_PATH=/opt/redash
+FILES_BASE_URL=https://raw.githubusercontent.com/EverythingMe/redash/docs_setup/setup/files/
+
+# Verify running as root:
+if [ "$(id -u)" != "0" ]; then
+    if [ $# -ne 0 ]; then
+        echo "Failed running with sudo. Exiting." 1>&2
+        exit 1
+    fi
+    echo "This script must be run as root. Trying to run with sudo."
+    sudo bash $0 --with-sudo
+    exit 0
+fi
+
+# Base packages
+apt-get update
+apt-get install -y python-pip python-dev nginx curl build-essential pwgen
+
+# redash user
+# TODO: check user doesn't exist yet?
+adduser --system --no-create-home --disabled-login --gecos "" redash
+
+# PostgreSQL
+pg_available=0
+psql --version || pg_available=$?
+if [ $pg_available -ne 0 ]; then
+    wget $FILES_BASE_URL"postgres_apt.sh" -O /tmp/postgres_apt.sh
+    bash /tmp/postgres_apt.sh
+    apt-get update
+    apt-get -y install postgresql-9.3 postgresql-server-dev-9.3
+fi
+
+add_service() {
+    service_name=$1
+    service_command="/etc/init.d/$service_name"
+
+    echo "Adding service: $service_name (/etc/init.d/$service_name)."
+    chmod +x $service_command
+
+    if command -v chkconfig >/dev/null 2>&1; then
+        # we're chkconfig, so lets add to chkconfig and put in runlevel 345
+        chkconfig --add $service_name && echo "Successfully added to chkconfig!"
+        chkconfig --level 345 $service_name on && echo "Successfully added to runlevels 345!"
+    elif command -v update-rc.d >/dev/null 2>&1; then
+        #if we're not a chkconfig box assume we're able to use update-rc.d
+        update-rc.d $service_name defaults && echo "Success!"
+    else
+        echo "No supported init tool found."
+    fi
+
+    $service_command start
+}
+
+# Redis
+redis_available=0
+redis-cli --version || redis_available=$?
+if [ $redis_available -ne 0 ]; then
+    wget http://download.redis.io/releases/redis-2.8.17.tar.gz
+    tar xzf redis-2.8.17.tar.gz
+    rm redis-2.8.17.tar.gz
+    cd redis-2.8.17
+    make
+    make install
+
+    # Setup process init & configuration
+
+    REDIS_PORT=6379
+    REDIS_CONFIG_FILE="/etc/redis/$REDIS_PORT.conf"
+    REDIS_LOG_FILE="/var/log/redis_$REDIS_PORT.log"
+    REDIS_DATA_DIR="/var/lib/redis/$REDIS_PORT"
+
+    mkdir -p `dirname "$REDIS_CONFIG_FILE"` || die "Could not create redis config directory"
+    mkdir -p `dirname "$REDIS_LOG_FILE"` || die "Could not create redis log dir"
+    mkdir -p "$REDIS_DATA_DIR" || die "Could not create redis data directory"
+
+    wget -O /etc/init.d/redis_6379 $FILES_BASE_URL"redis_init"
+    wget -O $REDIS_CONFIG_FILE $FILES_BASE_URL"redis.conf"
+
+    add_service "redis_$REDIS_PORT"
+
+    cd ..
+    rm -rf redis-2.8.17
+fi
+
+# Directories
+if [ ! -d "$REDASH_BASE_PATH" ]; then
+    sudo mkdir /opt/redash
+    sudo chown redash /opt/redash
+    sudo -u redash mkdir /opt/redash/logs
+fi
+
+# Default config file
+if [ ! -f "/opt/redash/.env" ]; then
+    sudo -u redash wget $FILES_BASE_URL"env" -O /opt/redash/.env
+fi
+
+# Install latest version
+LATEST_URL="https://github.com/EverythingMe/redash/releases/download/v0.4.0%2Bb563/redash.0.4.0.b563.tar.gz"
+VERSION_DIR=/opt/redash/redash.0.4.0.b563
+REDASH_TARBALL=/tmp/redash.tar.gz
+
+if [ ! -d "$VERSION_DIR" ]; then
+    sudo -u redash wget $LATEST_URL -O $REDASH_TARBALL
+    sudo -u redash mkdir $VERSION_DIR
+    sudo -u redash tar -C $VERSION_DIR -xvf $REDASH_TARBALL
+    ln -nfs $VERSION_DIR /opt/redash/current
+    ln -nfs /opt/redash/.env /opt/redash/current/.env
+
+    cd /opt/redash/current
+
+    # TODO: venv?
+    pip install -r requirements.txt
+fi
+
+# Create database / tables
+pg_user_exists=0
+sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redash'" | grep -q 1 || pg_user_exists=$?
+if [ $pg_user_exists -ne 0 ]; then
+    echo "Creating redash postgres user & database."
+    sudo -u postgres createuser redash --no-superuser --no-createdb --no-createrole
+    sudo -u postgres createdb redash --owner=redash
+
+    cd /opt/redash/current
+    sudo -u redash bin/run ./manage.py database create_tables
+fi
+
+# Create default admin user
+cd /opt/redash/current
+# TODO: make sure user created only once
+# TODO: generate temp password and print to screen
+sudo -u redash bin/run ./manage.py users create --admin --password admin "Admin" "admin"
+
+# Create re:dash read only pg user & setup data source
+pg_user_exists=0
+sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='redash_reader'" | grep -q 1 || pg_user_exists=$?
+if [ $pg_user_exists -ne 0 ]; then
+    echo "Creating redash reader postgres user."
+    REDASH_READER_PASSWORD=$(pwgen -1)
+    sudo -u postgres psql -c "CREATE ROLE redash_reader WITH PASSWORD '$REDASH_READER_PASSWORD' NOCREATEROLE NOCREATEDB NOSUPERUSER LOGIN"
+    sudo -u redash psql -c "grant select(id,name,type) ON data_sources to redash_reader;" redash
+    sudo -u redash psql -c "grant select on events, queries, dashboards, widgets, visualizations, query_results to redash_reader;" redash
+
+    cd /opt/redash/current
+    sudo -u redash bin/run ./manage.py ds new "re:dash metadata" "pg" "user=redash_reader password=$REDASH_READER_PASSWORD host=localhost dbname=redash"
+fi
+
+# BigQuery dependencies:
+apt-get install -y libffi-dev libssl-dev
+pip install google-api-python-client==1.2 pyOpenSSL==0.14 oauth2client==1.2
+
+# MySQL dependencies:
+apt-get install -y libmysqlclient-dev
+pip install MySQL-python==1.2.5
+
+# Setup supervisord + sysv init startup script
+sudo -u redash mkdir -p /opt/redash/supervisord
+pip install supervisor==3.1.2 # TODO: move to requirements.txt
+
+# Get supervisord startup script
+sudo -u redash wget -O /opt/redash/supervisord/supervisord.conf $FILES_BASE_URL"supervisord.conf"
+
+wget -O /etc/init.d/redash_supervisord $FILES_BASE_URL"redash_supervisord_init"
+add_service "redash_supervisord"
+
+# Nginx setup
+rm /etc/nginx/sites-enabled/default
+wget -O /etc/nginx/sites-available/redash $FILES_BASE_URL"nginx_redash_site"
+ln -nfs /etc/nginx/sites-available/redash /etc/nginx/sites-enabled/redash
+service nginx restart
diff --git a/setup/files/env b/setup/files/env
@@ -0,0 +1,9 @@
+export REDASH_CONNECTION_ADAPTER=pg
+export REDASH_CONNECTION_STRING="dbname=redash"
+export REDASH_STATIC_ASSETS_PATH="../rd_ui/dist/"
+export REDASH_LOG_LEVEL="INFO"
+export REDASH_WORKERS_COUNT=6
+export REDASH_REDIS_URL=redis://localhost:6379/1
+export REDASH_DATABASE_URL="postgresql://redash"
+export REDASH_COOKIE_SECRET=veryverysecret
+export REDASH_GOOGLE_APPS_DOMAIN=
diff --git a/setup/files/nginx_redash_site b/setup/files/nginx_redash_site
@@ -0,0 +1,20 @@
+upstream rd_servers {
+  server 127.0.0.1:5000;
+}
+
+server {
+  listen   80 default;
+
+  access_log /var/log/nginx/rd.access.log;
+
+  gzip on;
+  gzip_types *;
+  gzip_proxied any;
+
+  location / {
+    proxy_set_header Host $http_host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_pass       http://rd_servers;
+  }
+}
diff --git a/setup/files/postgres_apt.sh b/setup/files/postgres_apt.sh
@@ -0,0 +1,162 @@
+#!/bin/sh
+
+# script to add apt.postgresql.org to sources.list
+
+# from command line
+CODENAME="$1"
+# lsb_release is the best interface, but not always available
+if [ -z "$CODENAME" ]; then
+    CODENAME=$(lsb_release -cs 2>/dev/null)
+fi
+# parse os-release (unreliable, does not work on Ubuntu)
+if [ -z "$CODENAME" -a -f /etc/os-release ]; then
+    . /etc/os-release
+    # Debian: VERSION="7.0 (wheezy)"
+    # Ubuntu: VERSION="13.04, Raring Ringtail"
+    CODENAME=$(echo $VERSION | sed -ne 's/.*(\(.*\)).*/\1/')
+fi
+# guess from sources.list
+if [ -z "$CODENAME" ]; then
+    CODENAME=$(grep '^deb ' /etc/apt/sources.list | head -n1 | awk '{ print $3 }')
+fi
+# complain if no result yet
+if [ -z "$CODENAME" ]; then
+    cat <<EOF
+Could not determine the distribution codename. Please report this as a bug to
+[email protected]. As a workaround, you can call this script with
+the proper codename as parameter, e.g. "$0 squeeze".
+EOF
+    exit 1
+fi
+
+# errors are non-fatal above
+set -e
+
+cat <<EOF
+This script will enable the PostgreSQL APT repository on apt.postgresql.org on
+your system. The distribution codename used will be $CODENAME-pgdg.
+
+EOF
+
+case $CODENAME in
+    # known distributions
+    sid|wheezy|squeeze|lenny|etch) ;;
+    precise|lucid) ;;
+    *) # unknown distribution, verify on the web
+	DISTURL="http://apt.postgresql.org/pub/repos/apt/dists/"
+	if [ -x /usr/bin/curl ]; then
+	    DISTHTML=$(curl -s $DISTURL)
+	elif [ -x /usr/bin/wget ]; then
+	    DISTHTML=$(wget --quiet -O - $DISTURL)
+	fi
+	if [ "$DISTHTML" ]; then
+	    if ! echo "$DISTHTML" | grep -q "$CODENAME-pgdg"; then
+		cat <<EOF
+Your system is using the distribution codename $CODENAME, but $CODENAME-pgdg
+does not seem to be a valid distribution on
+$DISTURL
+
+We abort the installation here. If you want to use a distribution different
+from your system, you can call this script with an explicit codename, e.g.
+"$0 precise".
+
+Specifically, if you are using a non-LTS Ubuntu release, refer to
+https://wiki.postgresql.org/wiki/Apt/FAQ#I_am_using_a_non-LTS_release_of_Ubuntu
+
+For more information, refer to https://wiki.postgresql.org/wiki/Apt
+or ask on the mailing list for assistance: [email protected]
+EOF
+		exit 1
+	    fi
+	fi
+	;;
+esac
+
+echo "Writing /etc/apt/sources.list.d/pgdg.list ..."
+cat > /etc/apt/sources.list.d/pgdg.list <<EOF
+deb http://apt.postgresql.org/pub/repos/apt/ $CODENAME-pgdg main
+#deb-src http://apt.postgresql.org/pub/repos/apt/ $CODENAME-pgdg main
+EOF
+
+echo "Importing repository signing key ..."
+KEYRING="/etc/apt/trusted.gpg.d/apt.postgresql.org.gpg"
+test -e $KEYRING || touch $KEYRING
+apt-key --keyring $KEYRING add - <<EOF
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQINBE6XR8IBEACVdDKT2HEH1IyHzXkb4nIWAY7echjRxo7MTcj4vbXAyBKOfjja
+UrBEJWHN6fjKJXOYWXHLIYg0hOGeW9qcSiaa1/rYIbOzjfGfhE4x0Y+NJHS1db0V
+G6GUj3qXaeyqIJGS2z7m0Thy4Lgr/LpZlZ78Nf1fliSzBlMo1sV7PpP/7zUO+aA4
+bKa8Rio3weMXQOZgclzgeSdqtwKnyKTQdXY5MkH1QXyFIk1nTfWwyqpJjHlgtwMi
+c2cxjqG5nnV9rIYlTTjYG6RBglq0SmzF/raBnF4Lwjxq4qRqvRllBXdFu5+2pMfC
+IZ10HPRdqDCTN60DUix+BTzBUT30NzaLhZbOMT5RvQtvTVgWpeIn20i2NrPWNCUh
+hj490dKDLpK/v+A5/i8zPvN4c6MkDHi1FZfaoz3863dylUBR3Ip26oM0hHXf4/2U
+A/oA4pCl2W0hc4aNtozjKHkVjRx5Q8/hVYu+39csFWxo6YSB/KgIEw+0W8DiTII3
+RQj/OlD68ZDmGLyQPiJvaEtY9fDrcSpI0Esm0i4sjkNbuuh0Cvwwwqo5EF1zfkVj
+Tqz2REYQGMJGc5LUbIpk5sMHo1HWV038TWxlDRwtOdzw08zQA6BeWe9FOokRPeR2
+AqhyaJJwOZJodKZ76S+LDwFkTLzEKnYPCzkoRwLrEdNt1M7wQBThnC5z6wARAQAB
+tBxQb3N0Z3JlU1FMIERlYmlhbiBSZXBvc2l0b3J5iQI9BBMBCAAnAhsDBQsJCAcD
+BRUKCQgLBRYCAwEAAh4BAheABQJS6RUZBQkOhCctAAoJEH/MfUaszEz4zmQP/2ad
+HtuaXL5Xu3C3NGLha/aQb9iSJC8z5vN55HMCpsWlmslCBuEr+qR+oZvPkvwh0Io/
+8hQl/qN54DMNifRwVL2n2eG52yNERie9BrAMK2kNFZZCH4OxlMN0876BmDuNq2U6
+7vUtCv+pxT+g9R1LvlPgLCTjS3m+qMqUICJ310BMT2cpYlJx3YqXouFkdWBVurI0
+pGU/+QtydcJALz5eZbzlbYSPWbOm2ZSS2cLrCsVNFDOAbYLtUn955yXB5s4rIscE
+vTzBxPgID1iBknnPzdu2tCpk07yJleiupxI1yXstCtvhGCbiAbGFDaKzhgcAxSIX
+0ZPahpaYLdCkcoLlfgD+ar4K8veSK2LazrhO99O0onRG0p7zuXszXphO4E/WdbTO
+yDD35qCqYeAX6TaB+2l4kIdVqPgoXT/doWVLUK2NjZtd3JpMWI0OGYDFn2DAvgwP
+xqKEoGTOYuoWKssnwLlA/ZMETegak27gFAKfoQlmHjeA/PLC2KRYd6Wg2DSifhn+
+2MouoE4XFfeekVBQx98rOQ5NLwy/TYlsHXm1n0RW86ETN3chj/PPWjsi80t5oepx
+82azRoVu95LJUkHpPLYyqwfueoVzp2+B2hJU2Rg7w+cJq64TfeJG8hrc93MnSKIb
+zTvXfdPtvYdHhhA2LYu4+5mh5ASlAMJXD7zIOZt2iEYEEBEIAAYFAk6XSO4ACgkQ
+xa93SlhRC1qmjwCg9U7U+XN7Gc/dhY/eymJqmzUGT/gAn0guvoX75Y+BsZlI6dWn
+qaFU6N8HiQIcBBABCAAGBQJOl0kLAAoJEExaa6sS0qeuBfEP/3AnLrcKx+dFKERX
+o4NBCGWr+i1CnowupKS3rm2xLbmiB969szG5TxnOIvnjECqPz6skK3HkV3jTZaju
+v3sR6M2ItpnrncWuiLnYcCSDp9TEMpCWzTEgtrBlKdVuTNTeRGILeIcvqoZX5w+u
+i0eBvvbeRbHEyUsvOEnYjrqoAjqUJj5FUZtR1+V9fnZp8zDgpOSxx0LomnFdKnhj
+uyXAQlRCA6/roVNR9ruRjxTR5ubteZ9ubTsVYr2/eMYOjQ46LhAgR+3Alblu/WHB
+MR/9F9//RuOa43R5Sjx9TiFCYol+Ozk8XRt3QGweEH51YkSYY3oRbHBb2Fkql6N6
+YFqlLBL7/aiWnNmRDEs/cdpo9HpFsbjOv4RlsSXQfvvfOayHpT5nO1UQFzoyMVpJ
+615zwmQDJT5Qy7uvr2eQYRV9AXt8t/H+xjQsRZCc5YVmeAo91qIzI/tA2gtXik49
+6yeziZbfUvcZzuzjjxFExss4DSAwMgorvBeIbiz2k2qXukbqcTjB2XqAlZasd6Ll
+nLXpQdqDV3McYkP/MvttWh3w+J/woiBcA7yEI5e3YJk97uS6+ssbqLEd0CcdT+qz
++Waw0z/ZIU99Lfh2Qm77OT6vr//Zulw5ovjZVO2boRIcve7S97gQ4KC+G/+QaRS+
+VPZ67j5UMxqtT/Y4+NHcQGgwF/1iiQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYC
+AwEAAh4BAheABQJQeSssBQkDwxbfAAoJEH/MfUaszEz4bgkP/0AI0UgDgkNNqplA
+IpE/pkwem2jgGpJGKurh2xDu6j2ZL+BPzPhzyCeMHZwTXkkI373TXGQQP8dIa+RD
+HAZ3iijw4+ISdKWpziEUJjUk04UMPTlN+dYJt2EHLQDD0VLtX0yQC/wLmVEH/REp
+oclbVjZR/+ehwX2IxOIlXmkZJDSycl975FnSUjMAvyzty8P9DN0fIrQ7Ju+BfMOM
+TnUkOdp0kRUYez7pxbURJfkM0NxAP1geACI91aISBpFg3zxQs1d3MmUIhJ4wHvYB
+uaR7Fx1FkLAxWddre/OCYJBsjucE9uqc04rgKVjN5P/VfqNxyUoB+YZ+8Lk4t03p
+RBcD9XzcyOYlFLWXbcWxTn1jJ2QMqRIWi5lzZIOMw5B+OK9LLPX0dAwIFGr9WtuV
+J2zp+D4CBEMtn4Byh8EaQsttHeqAkpZoMlrEeNBDz2L7RquPQNmiuom15nb7xU/k
+7PGfqtkpBaaGBV9tJkdp7BdH27dZXx+uT+uHbpMXkRrXliHjWpAw+NGwADh/Pjmq
+ExlQSdgAiXy1TTOdzxKH7WrwMFGDK0fddKr8GH3f+Oq4eOoNRa6/UhTCmBPbryCS
+IA7EAd0Aae9YaLlOB+eTORg/F1EWLPm34kKSRtae3gfHuY2cdUmoDVnOF8C9hc0P
+bL65G4NWPt+fW7lIj+0+kF19s2PviQI9BBMBCAAnAhsDBQsJCAcDBRUKCQgLBRYC
+AwEAAh4BAheABQJRKm2VBQkINsBBAAoJEH/MfUaszEz4RTEP/1sQHyjHaUiAPaCA
+v8jw/3SaWP/g8qLjpY6ROjLnDMvwKwRAoxUwcIv4/TWDOMpwJN+CJIbjXsXNYvf9
+OX+UTOvq4iwi4ADrAAw2xw+Jomc6EsYla+hkN2FzGzhpXfZFfUsuphjY3FKL+4hX
+H+R8ucNwIz3yrkfc17MMn8yFNWFzm4omU9/JeeaafwUoLxlULL2zY7H3+QmxCl0u
+6t8VvlszdEFhemLHzVYRY0Ro/ISrR78CnANNsMIy3i11U5uvdeWVCoWV1BXNLzOD
+4+BIDbMB/Do8PQCWiliSGZi8lvmj/sKbumMFQonMQWOfQswTtqTyQ3yhUM1LaxK5
+PYq13rggi3rA8oq8SYb/KNCQL5pzACji4TRVK0kNpvtxJxe84X8+9IB1vhBvF/Ji
+/xDd/3VDNPY+k1a47cON0S8Qc8DA3mq4hRfcgvuWy7ZxoMY7AfSJOhleb9+PzRBB
+n9agYgMxZg1RUWZazQ5KuoJqbxpwOYVFja/stItNS4xsmi0lh2I4MNlBEDqnFLUx
+SvTDc22c3uJlWhzBM/f2jH19uUeqm4jaggob3iJvJmK+Q7Ns3WcfhuWwCnc1+58d
+iFAMRUCRBPeFS0qd56QGk1r97B6+3UfLUslCfaaA8IMOFvQSHJwDO87xWGyxeRTY
+IIP9up4xwgje9LB7fMxsSkCDTHOk
+=s3DI
+-----END PGP PUBLIC KEY BLOCK-----
+EOF
+
+echo "Running apt-get update ..."
+apt-get update
+
+cat <<EOF
+
+You can now start installing packages from apt.postgresql.org.
+
+Have a look at https://wiki.postgresql.org/wiki/Apt for more information;
+most notably the FAQ at https://wiki.postgresql.org/wiki/Apt/FAQ
+EOF