cmd: implement better migration handling

Dockerfile-demo

@@ -10,6 +10,8 @@ RUN glide install --skip-test -v
 ADD . .
 RUN go install .
 
+RUN /go/bin/hydra migrate sql
+
 ENTRYPOINT /go/bin/hydra host --dangerous-auto-logon --dangerous-force-http
 
 EXPOSE 4444

HISTORY.md

@@ -12,10 +12,14 @@ below 0.7.0**.
 
 To upgrade the database schemas, please run the following commands in exactly this order
 
+```sh
+$ hydra help migrate sql
+$ hydra help migrate ladon
 ```
-$ export DATABSE_URL=mysql://...
-$ hydra migrate sql
-$ hydra migrate ladon-0.6.0-to-0.7.0
+
+```sh
+$ hydra migrate sql mysql://...
+$ hydra migrate ladon 0.6.0 mysql://...
 ```
 
 ### Breaking changes

client/manager_sql.go

@@ -118,13 +118,13 @@ func (d *sqlData) ToClient() *Client {
 	}
 }
 
-func (s *SQLManager) CreateSchemas() error {
+func (s *SQLManager) CreateSchemas() (int, error) {
 	migrate.SetTable("hydra_client_migration")
 	n, err := migrate.Exec(s.DB.DB, s.DB.DriverName(), migrations, migrate.Up)
 	if err != nil {
-		return errors.Wrapf(err, "Could not migrate sql schema, applied %d migrations", n)
+		return 0, errors.Wrapf(err, "Could not migrate sql schema, applied %d migrations", n)
 	}
-	return nil
+	return n, nil
 }
 
 func (m *SQLManager) GetConcreteClient(id string) (*Client, error) {

client/manager_test.go

@@ -75,7 +75,7 @@ func TestMain(m *testing.M) {
 func connectToMySQL() {
 	var db = integration.ConnectToMySQL()
 	s := &SQLManager{DB: db, Hasher: &fosite.BCrypt{WorkFactor: 4}}
-	if err := s.CreateSchemas(); err != nil {
+	if _, err := s.CreateSchemas(); err != nil {
 		log.Fatalf("Could not create postgres schema: %v", err)
 	}
 
@@ -86,7 +86,7 @@ func connectToPG() {
 	var db = integration.ConnectToPostgres()
 	s := &SQLManager{DB: db, Hasher: &fosite.BCrypt{WorkFactor: 4}}
 
-	if err := s.CreateSchemas(); err != nil {
+	if _, err := s.CreateSchemas(); err != nil {
 		log.Fatalf("Could not create postgres schema: %v", err)
 	}
 

cmd/cli/handler.go

@@ -11,6 +11,7 @@ type Handler struct {
 	Warden     *WardenHandler
 	Revocation *RevocationHandler
 	Groups     *GroupHandler
+	Migration *MigrateHandler
 }
 
 func NewHandler(c *config.Config) *Handler {
@@ -21,5 +22,6 @@ func NewHandler(c *config.Config) *Handler {
 		Warden:     newWardenHandler(c),
 		Revocation: newRevocationHandler(c),
 		Groups:     newGroupHandler(c),
+		Migration: newMigrateHandler(c),
 	}
 }

cmd/cli/handler_migrate.go

@@ -0,0 +1,157 @@
+package cli
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"strings"
+	"github.com/jmoiron/sqlx"
+	ladon "github.com/ory/ladon/manager/sql"
+	"net/url"
+	"github.com/ory/hydra/pkg"
+	"time"
+	"github.com/pkg/errors"
+	"github.com/ory/hydra/config"
+	"github.com/ory/hydra/jwk"
+	"github.com/ory/hydra/warden/group"
+	"github.com/ory/hydra/client"
+	"github.com/ory/hydra/oauth2"
+	"os"
+)
+
+type MigrateHandler struct {
+	c *config.Config
+	M *jwk.HTTPManager
+}
+
+func newMigrateHandler(c *config.Config) *MigrateHandler {
+	return &MigrateHandler{
+		c: c,
+		M:      &jwk.HTTPManager{},
+	}
+}
+
+type schemaCreator interface {
+	CreateSchemas() (int, error)
+}
+
+func (h *MigrateHandler) connectToSql(dsn string) (*sqlx.DB, error) {
+	var db *sqlx.DB
+
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return nil, errors.Errorf("Could not parse DATABASE_URL: %s", err)
+	}
+
+	if err := pkg.Retry(h.c.GetLogger(), time.Second * 15, time.Minute * 2, func() error {
+		if u.Scheme == "mysql" {
+			dsn = strings.Replace(dsn, "mysql://", "", -1)
+		}
+
+		if db, err = sqlx.Open(u.Scheme, dsn); err != nil {
+			return errors.Errorf("Could not connect to SQL: %s", err)
+		} else if err := db.Ping(); err != nil {
+			return errors.Errorf("Could not connect to SQL: %s", err)
+		}
+
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+func (h *MigrateHandler) MigrateLadon050To060(cmd *cobra.Command, args []string) {
+	if len(args) != 2 {
+		fmt.Println(cmd.UsageString())
+		return
+	} else if args[0] != "0.6.0" {
+		fmt.Println(cmd.UsageString())
+		return
+	}
+
+	db, err := h.connectToSql(args[1])
+	if err != nil {
+		fmt.Printf("An error occurred while connecting to SQL: %s", err)
+		os.Exit(1)
+		return
+	}
+
+	if err := h.runMigrateLadon050To060(db); err != nil {
+		fmt.Printf("An error occurred while running the migrations: %s", err)
+		os.Exit(1)
+		return
+	}
+}
+
+func (h *MigrateHandler) runMigrateLadon050To060(db *sqlx.DB) error {
+	m := ladon.NewSQLManager(db, nil)
+	fmt.Printf("Applying `%s` SQL migrations.\n", "ladon")
+	if num, err := m.CreateSchemas("", "hydra_policy_migration"); err != nil {
+		return errors.Wrap(err, "Could not apply `ladon` SQL migrations")
+	} else {
+		fmt.Printf("Applied %d `%s` SQL migrations.\n", num, "ladon")
+	}
+
+	fmt.Println("Moving policies to new schema")
+	mm := ladon.SQLManagerMigrateFromMajor0Minor6ToMajor0Minor7{
+		DB:db,
+		SQLManager: m,
+	}
+	if err := mm.Migrate(); err != nil {
+		return errors.Wrap(err, "Could not move policies to new schema")
+	}
+	fmt.Println("Migration successful!")
+	return nil
+}
+
+func (h *MigrateHandler) MigrateSQL(cmd *cobra.Command, args []string) {
+	if len(args) == 0 {
+		fmt.Println(cmd.UsageString())
+		return
+	}
+
+	db, err := h.connectToSql(args[0])
+	if err != nil {
+		fmt.Printf("An error occurred while connecting to SQL: %s", err)
+		os.Exit(1)
+		return
+	}
+
+	if err := h.runMigrateSQL(db); err != nil {
+		fmt.Printf("An error occurred while running the migrations: %s", err)
+		os.Exit(1)
+		return
+	}
+	fmt.Println("Migration successful!")
+}
+
+func (h *MigrateHandler) runMigrateSQL(db *sqlx.DB) error {
+	var total int
+	fmt.Printf("Applying `%s` SQL migrations...\n", "ladon")
+	if num, err := ladon.NewSQLManager(db, nil).CreateSchemas("", "hydra_policy_migration"); err != nil {
+		return errors.Wrap(err, "Could not apply `ladon` SQL migrations")
+	} else {
+		fmt.Printf("Applied %d `%s` SQL migrations.\n", num, "ladon")
+		total += num
+	}
+
+	for k, m := range map[string]schemaCreator{
+		"client": &client.SQLManager{DB: db},
+		"oauth2": &oauth2.FositeSQLStore{DB: db},
+		"jwk": &jwk.SQLManager{DB: db},
+		"group": &group.SQLManager{DB: db},
+	} {
+		fmt.Printf("Applying `%s` SQL migrations...\n", k)
+		if num, err := m.CreateSchemas(); err != nil {
+			return errors.Wrapf(err, "Could not apply `%s` SQL migrations", k)
+		} else {
+			fmt.Printf("Applied %d `%s` SQL migrations.\n", num, k)
+			total += num
+		}
+	}
+
+	fmt.Printf("Migration successful! Applied a total of %d SQL migrations.\n", total)
+	return nil
+}

cmd/cli/handler_test.go

@@ -4,8 +4,63 @@ import (
 	"testing"
 
 	"github.com/ory/hydra/config"
+	"github.com/ory/hydra/integration"
+	"github.com/jmoiron/sqlx"
+	"os"
+	"github.com/stretchr/testify/assert"
+	lsql "github.com/ory/ladon/manager/sql"
+	"github.com/ory/ladon"
+	"github.com/pborman/uuid"
+	"github.com/stretchr/testify/require"
 )
 
+var db *sqlx.DB
+
+func TestMain(m *testing.M) {
+	db = integration.ConnectToPostgres()
+
+	code := m.Run()
+	integration.KillAll()
+	os.Exit(code)
+}
+
 func TestNewHandler(t *testing.T) {
 	_ = NewHandler(&config.Config{})
 }
+
+func TestMigrateHandlerSQL(t *testing.T) {
+	handler := newMigrateHandler(&config.Config{})
+
+	assert.NoError(t, handler.runMigrateSQL(db))
+
+	// create a few policies
+	m := lsql.SQLManagerMigrateFromMajor0Minor6ToMajor0Minor7{
+		DB: db,
+	}
+
+	// create some dummy policies
+	for _, p := range []*ladon.DefaultPolicy{
+		{
+			ID:          uuid.New(),
+			Description: "description",
+			Subjects:    []string{"user", "anonymous"},
+			Effect:      ladon.AllowAccess,
+			Resources:   []string{"article", "user"},
+			Actions:     []string{"create", "update"},
+			Conditions:  ladon.Conditions{},
+		},
+		{
+			ID:          uuid.New(),
+			Description: "description",
+			Subjects:    []string{},
+			Effect:      ladon.AllowAccess,
+			Resources:   []string{"<article|user>"},
+			Actions:     []string{"view"},
+			Conditions:  ladon.Conditions{},
+		},
+	} {
+		require.NoError(t, m.Create(p))
+	}
+
+	assert.NoError(t, handler.runMigrateLadon050To060(db))
+}

cmd/migrate.go

@@ -1,17 +1,3 @@
-// Copyright © 2017 NAME HERE <EMAIL ADDRESS>
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 package cmd
 
 import (
@@ -26,21 +12,10 @@ var migrateCmd = &cobra.Command{
 	Short: "Various migration helpers",
 	Run: func(cmd *cobra.Command, args []string) {
 		// TODO: Work your own magic here
-		fmt.Println("migrate called")
+		fmt.Print(cmd.UsageString())
 	},
 }
 
 func init() {
 	RootCmd.AddCommand(migrateCmd)
-
-	// Here you will define your flags and configuration settings.
-
-	// Cobra supports Persistent Flags which will work for this command
-	// and all subcommands, e.g.:
-	// migrateCmd.PersistentFlags().String("foo", "", "A help for foo")
-
-	// Cobra supports local flags which will only run when this command
-	// is called directly, e.g.:
-	// migrateCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
-
 }

cmd/migrate_ladon.go

@@ -0,0 +1,27 @@
+package cmd
+
+import "github.com/spf13/cobra"
+
+// migrateLadonCmd represents the ladon command
+var migrateLadonCmd = &cobra.Command{
+	Use:   "ladon 0.6.0 <database-url>",
+	Short: "Migrates Ladon SQL schema to version 0.6.0",
+	Long: `Hydra version 0.8.0 includes a breaking schema change from Ladon which was introduced
+with Ladon version 0.6.0. This script applies the neccessary migrations by copying data from the old tables
+to the new ones. This command might take some time, depending on how many policies are in your store.
+
+Do not run this command on a fresh installation.
+
+It is recommended to run this command close to the SQL instance (e.g. same subnet) instead of over the public internet.
+This decreases risk of failure and decreases time required.
+
+### WARNING ###
+
+Before running this command on an existing database, create a back up!
+`,
+	Run:cmdHandler.Migration.MigrateLadon050To060,
+}
+
+func init() {
+	migrateCmd.AddCommand(migrateLadonCmd)
+}