Skip to content

Commit

Permalink
Roll forward "Run Travis tests in Docker (letsencrypt#1830)" (letsenc…
Browse files Browse the repository at this point in the history
…rypt#1838)

That change broke the certbot tests because it switched to a MariaDB
10.1-specific syntax. certbot/certbot#3058 changes the certbot tests to use
Boulder's docker-compose.yml, so they will get MariaDB 10.1 automatically.
  • Loading branch information
jsha committed May 24, 2016
1 parent b954dcc commit 71e4af4
Show file tree
Hide file tree
Showing 17 changed files with 105 additions and 99 deletions.
2 changes: 0 additions & 2 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,2 @@
bin
tags
.git
test/js
31 changes: 8 additions & 23 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,23 +9,11 @@ addons:
- boulder
- boulder-mysql
- boulder-rabbitmq
apt:
packages:
- lsb-release
- python-dev
- python-virtualenv
- gcc
- libaugeas0
- libssl-dev
- libffi-dev
- ca-certificates
- rsyslog
mariadb: "10.0"

sudo: false
sudo: required

services:
- rabbitmq
- docker

matrix:
fast_finish: true
Expand All @@ -43,13 +31,6 @@ branches:
- release
- /^test-.*$/

# By providing our own install command we avoid Travis' default Go install
# command, which runs `go get`. We specifically want to avoid that because we
# want to ensure all our dependencies are vendored.
install:
- travis_retry test/travis-before-install.sh
- cd $GOPATH/src/github.com/letsencrypt/boulder

env:
global:
- PATH=$HOME/bin:$PATH # protoc gets installed here
Expand All @@ -61,6 +42,10 @@ env:
- RUN="integration" BOULDER_CONFIG="test/boulder-config-next.json"
- RUN="unit"

script:
- bash test.sh
install:
- docker-compose pull
- docker pull letsencrypt/boulder-tools
- docker-compose build

script:
- docker-compose run -e RUN="${RUN}" -e TRAVIS="${TRAVIS}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" boulder ./test.sh
38 changes: 14 additions & 24 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,34 +1,24 @@
FROM golang:1.5

MAINTAINER J.C. Jones "[email protected]"
MAINTAINER William Budington "[email protected]"

# Install dependencies packages
RUN apt-get update && apt-get install -y \
libltdl-dev \
mariadb-client-core-10.0 \
nodejs \
rsyslog \
softhsm \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

# Install port forwarder, database migration tool and go lint
RUN go get -v \
github.com/jsha/listenbuddy \
bitbucket.org/liamstask/goose/cmd/goose \
github.com/golang/lint/golint
FROM letsencrypt/boulder-tools:latest

# Boulder exposes its web application at port TCP 4000
EXPOSE 4000 4002 4003 8053 8055

ENV GO15VENDOREXPERIMENT 1
ENV GOBIN /go/src/github.com/letsencrypt/boulder/bin
ENV PATH /go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/
ENV GOPATH /go

WORKDIR /go/src/github.com/letsencrypt/boulder
RUN adduser --disabled-password --gecos "" --home /go/src/github.com/letsencrypt/boulder -q buser
RUN chown -R buser /go/

ENTRYPOINT [ "./test/entrypoint.sh" ]
WORKDIR /go/src/github.com/letsencrypt/boulder

# Copy in the Boulder sources
COPY . /go/src/github.com/letsencrypt/boulder
COPY . .
RUN mkdir bin
RUN go install ./cmd/rabbitmq-setup
COPY ./test/certbot /go/bin/

RUN GOBIN=/go/src/github.com/letsencrypt/boulder/bin go install ./...
RUN chown -R buser /go/

ENTRYPOINT [ "./test/entrypoint.sh" ]
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ setting](https://groups.google.com/forum/#!topic/binary-transparency/f-BI4o8HZW0
for better integrity guarantees when getting updates.

Boulder requires an installation of RabbitMQ, libtool-ltdl, goose, and
MariaDB 10 to work correctly. On Ubuntu and CentOS, you may have to
MariaDB 10.1 to work correctly. On Ubuntu and CentOS, you may have to
install RabbitMQ from https://rabbitmq.com/download.html to get a
recent version.

Expand Down
33 changes: 24 additions & 9 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,27 +1,42 @@
boulder:
build: .
dockerfile: Dockerfile
environment:
FAKE_DNS: 127.0.0.1
volumes:
# Cache built .a files for faster repeat runs
- /go/pkg/
- /tmp:/tmp
net: bridge
extra_hosts:
- le.wtf:127.0.0.1
- boulder:127.0.0.1
ports:
- 4000:4000
- 4002:4002
- 4003:4003
- 4000:4000 # ACME
- 4002:4002 # OCSP
- 4003:4003 # OCSP
- 4500:4500 # ct-test-srv
- 8000:8000 # debug ports
- 8001:8001
- 8002:8002
- 8003:8003
- 8004:8004
- 8055:8055 # dns-test-srv updates
- 9380:9380 # mail-test-srv
- 9381:9381 # mail-test-srv
links:
- bmysql:boulder-mysql
- brabbitmq:boulder-rabbitmq
extra_hosts:
- boulder:127.0.0.1
bmysql:
container_name: boulder-mysql
image: mariadb:10.0
image: mariadb:10.1
net: bridge
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
command: mysqld --bind-address=0.0.0.0
log_driver: none
brabbitmq:
container_name: boulder-rabbitmq
image: rabbitmq:3
net: bridge
environment:
RABBITMQ_NODE_IP_ADDRESS: "0.0.0.0"
log_driver: "none"
log_driver: none
7 changes: 5 additions & 2 deletions reloader/reloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ func (r *Reloader) Stop() {
r.stopChan <- struct{}{}
}

// A pointer we can override for testing.
var readFile = ioutil.ReadFile

// New loads the filename provided, and calls the callback. It then spawns a
// goroutine to check for updates to that file, calling the callback again with
// any new contents. The first load, and the first call to callback, are run
Expand All @@ -37,7 +40,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er
if err != nil {
return nil, err
}
b, err := ioutil.ReadFile(filename)
b, err := readFile(filename)
if err != nil {
return nil, err
}
Expand All @@ -58,7 +61,7 @@ func New(filename string, dataCallback func([]byte) error, errorCallback func(er
if !currentFileInfo.ModTime().After(fileInfo.ModTime()) {
continue
}
b, err := ioutil.ReadFile(filename)
b, err := readFile(filename)
if err != nil {
errorCallback(err)
continue
Expand Down
22 changes: 11 additions & 11 deletions reloader/reloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,16 @@ func TestNoStat(t *testing.T) {
func TestNoRead(t *testing.T) {
f, _ := ioutil.TempFile("", "test-no-read.txt")
defer os.Remove(f.Name())
err := f.Chmod(0)
if err != nil {
t.Fatalf("failed to chmod file: %s", err)
oldReadFile := readFile
readFile = func(string) ([]byte, error) {
return nil, fmt.Errorf("read failed")
}
_, err = New(f.Name(), noop, testErrCb(t))
_, err := New(f.Name(), noop, testErrCb(t))
if err == nil {
t.Fatalf("Expected New to return error when permission denied.")
readFile = oldReadFile
}
readFile = oldReadFile
}

func TestFirstError(t *testing.T) {
Expand Down Expand Up @@ -182,10 +184,11 @@ func TestReloadFailure(t *testing.T) {

time.Sleep(15 * time.Millisecond)
// Create a file with no permissions
err = ioutil.WriteFile(filename, []byte("second body"), 0)
if err != nil {
t.Fatal(err)
oldReadFile := readFile
readFile = func(string) ([]byte, error) {
return nil, fmt.Errorf("permisssion denied")
}

fakeTick <- time.Now()
select {
case r := <-reloads:
Expand All @@ -195,11 +198,8 @@ func TestReloadFailure(t *testing.T) {
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for reload")
}
readFile = oldReadFile

err = os.Remove(filename)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filename, []byte("third body"), 0644)
if err != nil {
t.Fatal(err)
Expand Down
21 changes: 7 additions & 14 deletions test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,6 @@ function die() {
exit 1
}

function build_certbot() {
run git clone \
https://www.github.com/certbot/certbot.git \
$CERTBOT_PATH || exit 1
cd $CERTBOT_PATH
run ./tools/venv.sh
cd -
}

function run_unit_tests() {
if [ "${TRAVIS}" == "true" ]; then

Expand Down Expand Up @@ -188,18 +179,20 @@ if [[ "$RUN" =~ "integration" ]] ; then
start_context "integration"

if [ -z "$CERTBOT_PATH" ]; then
export CERTBOT_PATH=$(mktemp -d -t leXXXX)
export CERTBOT_PATH=$(mktemp -d -t cbpXXXX)
echo "------------------------------------------------"
echo "--- Checking out letsencrypt client is slow. ---"
echo "--- Recommend setting \$CERTBOT_PATH to ---"
echo "--- client repo with initialized virtualenv ---"
echo "------------------------------------------------"
build_certbot
elif [ ! -d "${CERTBOT_PATH}" ]; then
build_certbot
run git clone \
https://www.github.com/certbot/certbot.git \
$CERTBOT_PATH || exit 1
fi

source ${CERTBOT_PATH}/venv/bin/activate
if ! type certbot >/dev/null 2>/dev/null; then
source ${CERTBOT_PATH}/${VENV_NAME:-venv}/bin/activate
fi

python test/integration-test.py --all
if [ "$?" != 0 ]; then
Expand Down
4 changes: 4 additions & 0 deletions test/certbot
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/bin/bash
#
# Temporary shim until the letsencrypt Debian package ships `certbot`
exec letsencrypt "$@"
3 changes: 0 additions & 3 deletions test/create_db.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@ fi
# to the format we use in production, MIXED.
mysql $dbconn -e "SET GLOBAL binlog_format = 'MIXED';"

# Drop all users to get a fresh start
mysql $dbconn < test/drop_users.sql

for dbenv in $DBENVS; do
(
db="boulder_sa_${dbenv}"
Expand Down
2 changes: 1 addition & 1 deletion test/ct-test-srv/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func main() {

is := integrationSrv{key: key}
s := &http.Server{
Addr: "localhost:4500",
Addr: "0.0.0.0:4500",
Handler: http.HandlerFunc(is.handler),
}
log.Fatal(s.ListenAndServe())
Expand Down
4 changes: 4 additions & 0 deletions test/docker-environment
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
PATH=/go/bin:/go/src/github.com/letsencrypt/boulder/bin:/usr/local/go/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin/
GOPATH=/go
GOBIN=/go/src/github.com/letsencrypt/boulder/bin
GO15VENDOREXPERIMENT=1
5 changes: 5 additions & 0 deletions test/drop_users.sql
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
-- Note that dropping a non-existing user produces an error that aborts the
-- script, so we first grant a harmless privilege to each user to ensure it
-- exists.

USE mysql;

GRANT USAGE ON *.* TO 'policy'@'localhost';
DROP USER 'policy'@'localhost';
GRANT USAGE ON *.* TO 'sa'@'localhost';
Expand All @@ -21,3 +24,5 @@ GRANT USAGE ON *.* TO 'cert_checker'@'localhost';
DROP USER 'cert_checker'@'localhost';
GRANT USAGE ON *.* TO 'backfiller'@'localhost';
DROP USER 'backfiller'@'localhost';
GRANT USAGE ON *.* TO 'test_setup'@'localhost';
DROP USER 'test_setup'@'localhost';
6 changes: 5 additions & 1 deletion test/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,14 @@ wait_tcp_port boulder-rabbitmq 5672
MYSQL_CONTAINER=1 $DIR/create_db.sh

# Set up rabbitmq exchange
go run cmd/rabbitmq-setup/main.go -server amqp://boulder-rabbitmq
rabbitmq-setup -server amqp://boulder-rabbitmq

if [[ $# -eq 0 ]]; then
exec ./start.py
fi

# TODO(jsha): Change to an unprivileged user before running commands. Currently,
# running as an unprivileged user causes the certbot integration test to fail
# during the test of the manual plugin. There's a call to killpg in there that
# kills the whole test, but only when run under `su buser -c "..."`
exec $@
6 changes: 3 additions & 3 deletions test/mail-test-srv/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
blog "github.com/letsencrypt/boulder/log"
)

var apiPort = flag.String("http", "9381", "http port to listen on")
var listenAPI = flag.String("http", "0.0.0.0:9381", "http port to listen on")

type rcvdMail struct {
From string
Expand Down Expand Up @@ -162,15 +162,15 @@ func serveSMTP(l net.Listener) error {
}

func main() {
l, err := net.Listen("tcp", ":9380")
l, err := net.Listen("tcp", "0.0.0.0:9380")
if err != nil {
log.Fatalln("Couldn't bind for SMTP", err)
}
defer l.Close()

setupHTTP(http.DefaultServeMux)
go func() {
err := http.ListenAndServe(":"+*apiPort, http.DefaultServeMux)
err := http.ListenAndServe(*listenAPI, http.DefaultServeMux)
if err != nil {
log.Fatalln("Couldn't start HTTP server", err)
}
Expand Down
2 changes: 1 addition & 1 deletion test/run-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ if [[ "$(is_running boulder-mysql)" != "true" ]]; then
docker run -d \
-e MYSQL_ALLOW_EMPTY_PASSWORD=yes \
--name boulder-mysql \
mariadb:10.0 mysqld --bind-address=0.0.0.0
mariadb:10.1 mysqld --bind-address=0.0.0.0
fi

if [[ "$(is_running boulder-rabbitmq)" != "true" ]]; then
Expand Down
Loading

0 comments on commit 71e4af4

Please sign in to comment.