Skip to content

Commit

Permalink
Merge pull request ethereum#3226 from karalabe/azure-gethstore
Browse files Browse the repository at this point in the history
travis, build: implement uploading archives to azure
  • Loading branch information
fjl authored Nov 3, 2016
2 parents 3914e82 + 2f9f2cb commit d0c820a
Show file tree
Hide file tree
Showing 60 changed files with 11,437 additions and 44 deletions.
26 changes: 22 additions & 4 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ matrix:
- os: linux
dist: trusty
go: 1.5.4
env:
- GO15VENDOREXPERIMENT=1
- os: linux
dist: trusty
go: 1.6.2
Expand All @@ -15,27 +17,43 @@ matrix:
- os: osx
go: 1.7

# This builder does the PPA upload (and nothing else).
# This builder does the Ubuntu PPA and Linux Azure uploads
- os: linux
dist: trusty
go: 1.7
env: PPA
env:
- ubuntu-ppa
- azure-linux
addons:
apt:
packages:
- gcc-multilib
- devscripts
- debhelper
- dput
script:
- go run build/ci.go debsrc -signer "Felix Lange (Geth CI Testing Key) <[email protected]>" -upload ppa:lp-fjl/geth-ci-testing
- go run build/ci.go install
- go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
- go run build/ci.go install -arch 386
- go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds

# This builder does the OSX Azure uploads
- os: osx
go: 1.7
env:
- azure-osx
script:
- go run build/ci.go install
- go run build/ci.go archive -type zip -signer OSX_SIGNING_KEY -upload gethstore/builds
- go run build/ci.go install -arch 386
- go run build/ci.go archive -arch 386 -type zip -signer OSX_SIGNING_KEY -upload gethstore/builds

install:
- go get golang.org/x/tools/cmd/cover
script:
- go run build/ci.go install
- go run build/ci.go test -coverage -vet
after_success:
# - go run build/ci.go archive -type tar

notifications:
webhooks:
Expand Down
110 changes: 95 additions & 15 deletions build/ci.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@ Usage: go run ci.go <command> <command flags/arguments>
Available commands are:
install [ packages... ] -- builds packages and executables
test [ -coverage ] [ -vet ] [ packages... ] -- runs the tests
archive [ -type zip|tar ] -- archives build artefacts
importkeys -- imports signing keys from env
debsrc [ -sign key-id ] [ -upload dest ] -- creates a debian source package
xgo [ options ] -- cross builds according to options
install [-arch architecture] [ packages... ] -- builds packages and executables
test [ -coverage ] [ -vet ] [ packages... ] -- runs the tests
archive [-arch architecture] [ -type zip|tar ] [ -signer key-envvar ] [ -upload dest ] -- archives build artefacts
importkeys -- imports signing keys from env
debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package
xgo [ options ] -- cross builds according to options
For all commands, -n prevents execution of external programs (dry run mode).
Expand All @@ -40,6 +40,8 @@ import (
"encoding/base64"
"flag"
"fmt"
"go/parser"
"go/token"
"io/ioutil"
"log"
"os"
Expand All @@ -49,7 +51,7 @@ import (
"strings"
"time"

"../internal/build"
"github.com/ethereum/go-ethereum/internal/build"
)

var (
Expand Down Expand Up @@ -130,6 +132,9 @@ func main() {
// Compiling

func doInstall(cmdline []string) {
var (
arch = flag.String("arch", "", "Architecture to cross build for")
)
flag.CommandLine.Parse(cmdline)
env := build.Env()

Expand All @@ -146,11 +151,38 @@ func doInstall(cmdline []string) {
if flag.NArg() > 0 {
packages = flag.Args()
}

goinstall := goTool("install", buildFlags(env)...)
if *arch == "" || *arch == runtime.GOARCH {
goinstall := goTool("install", buildFlags(env)...)
goinstall.Args = append(goinstall.Args, "-v")
goinstall.Args = append(goinstall.Args, packages...)
build.MustRun(goinstall)
return
}
// Seems we are cross compiling, work around forbidden GOBIN
goinstall := goToolArch(*arch, "install", buildFlags(env)...)
goinstall.Args = append(goinstall.Args, "-v")
goinstall.Args = append(goinstall.Args, []string{"-buildmode", "archive"}...)
goinstall.Args = append(goinstall.Args, packages...)
build.MustRun(goinstall)

if cmds, err := ioutil.ReadDir("cmd"); err == nil {
for _, cmd := range cmds {
pkgs, err := parser.ParseDir(token.NewFileSet(), filepath.Join(".", "cmd", cmd.Name()), nil, parser.PackageClauseOnly)
if err != nil {
log.Fatal(err)
}
for name, _ := range pkgs {
if name == "main" {
gobuild := goToolArch(*arch, "build", buildFlags(env)...)
gobuild.Args = append(gobuild.Args, "-v")
gobuild.Args = append(gobuild.Args, []string{"-o", filepath.Join(GOBIN, cmd.Name())}...)
gobuild.Args = append(gobuild.Args, "."+string(filepath.Separator)+filepath.Join("cmd", cmd.Name()))
build.MustRun(gobuild)
break
}
}
}
}
}

func buildFlags(env build.Environment) (flags []string) {
Expand All @@ -173,13 +205,22 @@ func buildFlags(env build.Environment) (flags []string) {
}

func goTool(subcmd string, args ...string) *exec.Cmd {
return goToolArch(runtime.GOARCH, subcmd, args...)
}

func goToolArch(arch string, subcmd string, args ...string) *exec.Cmd {
gocmd := filepath.Join(runtime.GOROOT(), "bin", "go")
cmd := exec.Command(gocmd, subcmd)
cmd.Args = append(cmd.Args, args...)
cmd.Env = []string{
"GO15VENDOREXPERIMENT=1",
"GOPATH=" + build.GOPATH(),
"GOBIN=" + GOBIN,
}
if arch == "" || arch == runtime.GOARCH {
cmd.Env = append(cmd.Env, "GOBIN="+GOBIN)
} else {
cmd.Env = append(cmd.Env, "CGO_ENABLED=1")
cmd.Env = append(cmd.Env, "GOARCH="+arch)
}
for _, e := range os.Environ() {
if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") {
Expand Down Expand Up @@ -239,8 +280,11 @@ func doTest(cmdline []string) {

func doArchive(cmdline []string) {
var (
atype = flag.String("type", "zip", "Type of archive to write (zip|tar)")
ext string
arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging")
atype = flag.String("type", "zip", "Type of archive to write (zip|tar)")
signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`)
upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`)
ext string
)
flag.CommandLine.Parse(cmdline)
switch *atype {
Expand All @@ -255,25 +299,61 @@ func doArchive(cmdline []string) {
env := build.Env()
maybeSkipArchive(env)

base := archiveBasename(env)
base := archiveBasename(*arch, env)
if err := build.WriteArchive("geth-"+base, ext, gethArchiveFiles); err != nil {
log.Fatal(err)
}
if err := build.WriteArchive("geth-alltools-"+base, ext, allToolsArchiveFiles); err != nil {
log.Fatal(err)
}

for _, archive := range []string{"geth-" + base + ext, "geth-alltools-" + base + ext} {
if err := archiveUpload(archive, *upload, *signer); err != nil {
log.Fatal(err)
}
}
}

func archiveBasename(env build.Environment) string {
func archiveBasename(arch string, env build.Environment) string {
// date := time.Now().UTC().Format("200601021504")
platform := runtime.GOOS + "-" + runtime.GOARCH
platform := runtime.GOOS + "-" + arch
archive := platform + "-" + build.VERSION()
if env.Commit != "" {
archive += "-" + env.Commit[:8]
}
return archive
}

func archiveUpload(archive string, blobstore string, signer string) error {
// If signing was requested, generate the signature files
if signer != "" {
pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer))
if err != nil {
return fmt.Errorf("invalid base64 %s", signer)
}
if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil {
return err
}
}
// If uploading to Azure was requested, push the archive possibly with its signature
if blobstore != "" {
auth := build.AzureBlobstoreConfig{
Account: strings.Split(blobstore, "/")[0],
Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"),
Container: strings.SplitN(blobstore, "/", 2)[1],
}
if err := build.AzureBlobstoreUpload(archive, archive, auth); err != nil {
return err
}
if signer != "" {
if err := build.AzureBlobstoreUpload(archive+".asc", archive+".asc", auth); err != nil {
return err
}
}
}
return nil
}

// skips archiving for some build configurations.
func maybeSkipArchive(env build.Environment) {
if env.IsPullRequest {
Expand Down
58 changes: 58 additions & 0 deletions internal/build/azure.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package build

import (
"os"

"github.com/Azure/azure-sdk-for-go/storage"
)

// AzureBlobstoreConfig is an authentication and configuration struct containing
// the data needed by the Azure SDK to interact with a speicifc container in the
// blobstore.
type AzureBlobstoreConfig struct {
Account string // Account name to authorize API requests with
Token string // Access token for the above account
Container string // Blob container to upload files into
}

// AzureBlobstoreUpload uploads a local file to the Azure Blob Storage. Note, this
// method assumes a max file size of 64MB (Azure limitation). Larger files will
// need a multi API call approach implemented.
//
// See: https://msdn.microsoft.com/en-us/library/azure/dd179451.aspx#Anchor_3
func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig) error {
// Create an authenticated client against the Azure cloud
rawClient, err := storage.NewBasicClient(config.Account, config.Token)
if err != nil {
return err
}
client := rawClient.GetBlobService()

// Stream the file to upload into the designated blobstore container
in, err := os.Open(path)
if err != nil {
return err
}
defer in.Close()

info, err := in.Stat()
if err != nil {
return err
}
return client.CreateBlockBlobFromReader(config.Container, name, uint64(info.Size()), in, nil)
}
58 changes: 58 additions & 0 deletions internal/build/pgp.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

// signFile reads the contents of an input file and signs it (in armored format)
// with the key provided, placing the signature into the output file.

package build

import (
"bytes"
"fmt"
"os"

"golang.org/x/crypto/openpgp"
)

// PGPSignFile parses a PGP private key from the specified string and creates a
// signature file into the output parameter of the input file.
//
// Note, this method assumes a single key will be container in the pgpkey arg,
// furthermore that it is in armored format.
func PGPSignFile(input string, output string, pgpkey string) error {
// Parse the keyring and make sure we only have a single private key in it
keys, err := openpgp.ReadArmoredKeyRing(bytes.NewBufferString(pgpkey))
if err != nil {
return err
}
if len(keys) != 1 {
return fmt.Errorf("key count mismatch: have %d, want %d", len(keys), 1)
}
// Create the input and output streams for signing
in, err := os.Open(input)
if err != nil {
return err
}
defer in.Close()

out, err := os.Create(output)
if err != nil {
return err
}
defer out.Close()

// Generate the signature and return
return openpgp.ArmoredDetachSignText(out, keys[0], in, nil)
}
Loading

0 comments on commit d0c820a

Please sign in to comment.