Opt out of binlogs

go/base/context.go

@@ -148,6 +148,7 @@ type MigrationContext struct {
 	InitiallyDropOldTable        bool
 	InitiallyDropGhostTable      bool
 	TimestampOldTable            bool // Should old table name include a timestamp
+	UseBinLog                    bool
 	CutOverType                  CutOver
 	ReplicaServerId              uint
 
@@ -248,6 +249,7 @@ func NewMigrationContext() *MigrationContext {
 		pointOfInterestTimeMutex:            &sync.Mutex{},
 		ColumnRenameMap:                     make(map[string]string),
 		PanicAbort:                          make(chan error),
+		UseBinLog:                           true,
 	}
 }
 

go/cmd/gh-ost/main.go

@@ -90,6 +90,7 @@ func main() {
 	cutOver := flag.String("cut-over", "atomic", "choose cut-over type (default|atomic, two-step)")
 	flag.BoolVar(&migrationContext.ForceNamedCutOverCommand, "force-named-cut-over", false, "When true, the 'unpostpone|cut-over' interactive command must name the migrated table")
 	flag.BoolVar(&migrationContext.ForceNamedPanicCommand, "force-named-panic", false, "When true, the 'panic' interactive command must name the migrated table")
+	flag.BoolVar(&migrationContext.UseBinLog, "sql-log-bin", true, "When true, gh-ost will write to the binlog for all operations that perform --test-on-replica, --migration-on-replica, and --allow-on-master")
 
 	flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running")
 	flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges")
@@ -220,6 +221,11 @@ func main() {
 	if *replicationLagQuery != "" {
 		log.Warningf("--replication-lag-query is deprecated")
 	}
+	if !migrationContext.UseBinLog && !(migrationContext.TestOnReplica ||
+		migrationContext.MigrateOnReplica ||
+		migrationContext.AllowedRunningOnMaster) {
+		log.Fatalf("--sql-log-bin=false is only available for --test-on-replica, --migrate-on-replica, or --allow-on-master")
+	}
 
 	switch *cutOver {
 	case "atomic", "default", "":

go/logic/applier.go

@@ -7,7 +7,9 @@ package logic
 
 import (
 	gosql "database/sql"
+	"errors"
 	"fmt"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -57,6 +59,7 @@ type Applier struct {
 	singletonDB       *gosql.DB
 	migrationContext  *base.MigrationContext
 	finishedMigrating int64
+	pristineGTIDSet   GTIDSet
 }
 
 func NewApplier(migrationContext *base.MigrationContext) *Applier {
@@ -85,6 +88,10 @@ func (this *Applier) InitDBConnections() (err error) {
 	if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig, this.migrationContext); err != nil {
 		return err
 	}
+	if !this.migrationContext.UseBinLog && strings.HasPrefix(version, "5.5.") {
+		return fmt.Errorf("SQL log bin is required for any MySQL version < 5.6. Current version: %+v", version)
+	}
+
 	this.migrationContext.ApplierMySQLVersion = version
 	if err := this.validateAndReadTimeZone(); err != nil {
 		return err
@@ -1043,3 +1050,97 @@ func (this *Applier) Teardown() {
 	this.singletonDB.Close()
 	atomic.StoreInt64(&this.finishedMigrating, 1)
 }
+
+func (this *Applier) SaveExistingGTIDs() error {
+	query := `select /* gh-ost */ @@global.server_uuid as server_uuid, @@global.gtid_executed as gtid_executed`
+	err := sqlutils.QueryRowsMap(this.singletonDB, query, func(rowMap sqlutils.RowMap) error {
+		this.pristineGTIDSet = NewGTIDSet(
+			rowMap.GetString("server_uuid"),
+			rowMap.GetString("gtid_executed"),
+		)
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (this *Applier) purgeBinaryLogs() error {
+	lastLog := ""
+	if err := sqlutils.QueryRowsMap(this.singletonDB, `SHOW BINARY LOGS`, func(rowMap sqlutils.RowMap) error {
+		lastLog = rowMap.GetString("Log_name")
+		return nil
+	}); err != nil {
+		return err
+	}
+	if lastLog == "" {
+		return errors.New("No binary logs found!")
+	}
+	log.Infof("Purging Binary Logs to '%s'", lastLog)
+	query := fmt.Sprintf(`PURGE BINARY LOGS TO '%s'`, lastLog)
+	if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (this *Applier) PurgeGTIDs() error {
+	err := this.purgeBinaryLogs()
+	if err != nil {
+		return err
+	}
+
+	slaveRunning := true
+	if err := sqlutils.QueryRowsMap(this.singletonDB, `SHOW SLAVE STATUS`, func(rowMap sqlutils.RowMap) error {
+		if rowMap.GetString("Slave_IO_Running") != "Yes" || rowMap.GetString("Slave_SQL_Running") != "Yes" {
+			slaveRunning = false
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	if slaveRunning {
+		if _, err := sqlutils.ExecNoPrepare(this.singletonDB, `STOP SLAVE`); err != nil {
+			return err
+		}
+	}
+
+	var gtidExecuted string
+	if err := sqlutils.QueryRowsMap(this.singletonDB,
+		`select /* gh-ost */ @@global.gtid_executed as gtid_executed`,
+		func(rowMap sqlutils.RowMap) error {
+			gtidExecuted = rowMap.GetString("gtid_executed")
+			return nil
+		}); err != nil {
+		return err
+	}
+
+	log.Infof("Setting GTID_PURGED")
+	if _, err := sqlutils.ExecNoPrepare(this.singletonDB, `RESET MASTER`); err != nil {
+		return err
+	}
+	if _, err := sqlutils.ExecNoPrepare(this.singletonDB,
+		fmt.Sprintf(`SET GLOBAL GTID_PURGED='%s'`, this.pristineGTIDSet.Revert(gtidExecuted))); err != nil {
+		return err
+	}
+	if _, err := sqlutils.ExecNoPrepare(this.singletonDB, `FLUSH BINARY LOGS`); err != nil {
+		return err
+	}
+
+	err = this.purgeBinaryLogs()
+	if err != nil {
+		return err
+	}
+
+	if slaveRunning {
+		// Restart slave after
+		if _, err := sqlutils.ExecNoPrepare(this.singletonDB, `START SLAVE`); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

go/logic/gtidset.go

@@ -0,0 +1,61 @@
+/*
+	Copyright 2019 Zach Moazeni.
+	See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package logic
+
+import (
+	"strings"
+)
+
+type GTIDSet struct {
+	server_uuid   string
+	gtid_executed string
+}
+
+func NewGTIDSet(server_uuid, gtid_executed string) GTIDSet {
+	return GTIDSet{server_uuid, gtid_executed}
+}
+
+func (set *GTIDSet) originalServerGTIDs() (string, bool) {
+	if !strings.Contains(set.gtid_executed, set.server_uuid) {
+		return "", false
+	}
+	gtids := strings.Split(set.gtid_executed, ",")
+	for _, gtid := range gtids {
+		if strings.Contains(gtid, set.server_uuid) {
+			return strings.TrimSpace(gtid), true
+		}
+	}
+
+	// Shouldn't run into this. Only necessary for type checker
+	return "", false
+}
+
+func (set *GTIDSet) Revert(current_gtid_executed string) string {
+	if !strings.Contains(current_gtid_executed, set.server_uuid) {
+		return current_gtid_executed
+	}
+	gtids := strings.Split(current_gtid_executed, ",")
+	newGtids := make([]string, 0, len(gtids))
+	originalServerGTIDs, found := set.originalServerGTIDs()
+	if found {
+		// revert back to original gtid_executed JUST for the server_uuid
+		for _, gtid := range gtids {
+			if strings.Contains(gtid, set.server_uuid) {
+				newGtids = append(newGtids, originalServerGTIDs)
+			} else {
+				newGtids = append(newGtids, strings.TrimSpace(gtid))
+			}
+		}
+	} else {
+		// trim server_uuid current_gtid_executed out of the set and return
+		for _, gtid := range gtids {
+			if !strings.Contains(gtid, set.server_uuid) {
+				newGtids = append(newGtids, strings.TrimSpace(gtid))
+			}
+		}
+	}
+	return strings.Join(newGtids, ",\n")
+}

go/logic/gtidset_test.go

@@ -0,0 +1,35 @@
+/*
+	Copyright 2019 Zach Moazeni.
+	See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package logic
+
+import (
+	"testing"
+
+	test "github.com/outbrain/golib/tests"
+)
+
+func TestRevert(t *testing.T) {
+	{
+		set := NewGTIDSet("00005722-2222-2222-2222-222222222222", "00005721-1111-1111-1111-111111111111:1-52")
+		newGTIDs := set.Revert("00005721-1111-1111-1111-111111111111:1-54,\n00005722-2222-2222-2222-222222222222:1-7")
+		test.S(t).ExpectEquals(newGTIDs, "00005721-1111-1111-1111-111111111111:1-54")
+	}
+	{
+		set := NewGTIDSet("00005722-2222-2222-2222-222222222222", "00005721-1111-1111-1111-111111111111:1-52")
+		newGTIDs := set.Revert("00005721-1111-1111-1111-111111111111:1-54")
+		test.S(t).ExpectEquals(newGTIDs, "00005721-1111-1111-1111-111111111111:1-54")
+	}
+	{
+		set := NewGTIDSet("00005722-2222-2222-2222-222222222222", "00005721-1111-1111-1111-111111111111:1-52,\n00005722-2222-2222-2222-222222222222:1-7")
+		newGTIDs := set.Revert("00005721-1111-1111-1111-111111111111:1-100,\n00005722-2222-2222-2222-222222222222:1-80")
+		test.S(t).ExpectEquals(newGTIDs, "00005721-1111-1111-1111-111111111111:1-100,\n00005722-2222-2222-2222-222222222222:1-7")
+	}
+	{
+		set := NewGTIDSet("00005722-2222-2222-2222-222222222222", "00005721-1111-1111-1111-111111111111:1-52,\n00005722-2222-2222-2222-222222222222:1-7")
+		newGTIDs := set.Revert("00005721-1111-1111-1111-111111111111:1-100,\n00005722-2222-2222-2222-222222222222:1-80,\n00005722-3333-3333-3333-333333333333:1-5")
+		test.S(t).ExpectEquals(newGTIDs, "00005721-1111-1111-1111-111111111111:1-100,\n00005722-2222-2222-2222-222222222222:1-7,\n00005722-3333-3333-3333-333333333333:1-5")
+	}
+}

go/logic/migrator.go

@@ -1050,6 +1050,13 @@ func (this *Migrator) initiateApplier() error {
 	if err := this.applier.InitDBConnections(); err != nil {
 		return err
 	}
+
+	if !this.migrationContext.UseBinLog {
+		if err := this.applier.SaveExistingGTIDs(); err != nil {
+			return err
+		}
+	}
+
 	if err := this.applier.ValidateOrDropExistingTables(); err != nil {
 		return err
 	}
@@ -1289,6 +1296,12 @@ func (this *Migrator) finalCleanup() error {
 		}
 	}
 
+	if !this.migrationContext.UseBinLog {
+		if err := this.applier.PurgeGTIDs(); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

localtests/sql-log-bin-off-replica-gtids/after.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# shellcheck disable=SC2168
+
+sleep 1 # let any any replication catch up
+
+local master_server_uuid replica_server_uuid gtid_executed gtid_purged
+
+master_server_uuid=$(gh-ost-test-mysql-master test -e "SELECT @@global.server_uuid" -ss)
+replica_server_uuid=$(gh-ost-test-mysql-replica test -e "SELECT @@global.server_uuid" -ss)
+gtid_executed=$(gh-ost-test-mysql-replica test -e "SELECT @@global.gtid_executed" -ss)
+gtid_purged=$(gh-ost-test-mysql-replica test -e "SELECT @@global.gtid_purged" -ss)
+
+if ! echo "$gtid_executed" | grep -q "$replica_server_uuid" ; then
+  echo
+  echo "ERROR gtid_executed is missing original gh-ost gtids"
+  # shellcheck disable=SC2154
+  echo "prior gtid_executed:   $gtid_executed_pristine"
+  echo "current gtid_executed: $gtid_executed"
+  return 1
+fi
+
+if [ "$gtid_executed" != "$gtid_purged" ] ; then
+  echo
+  echo "ERROR gtid_executed and gtid_purged don't match"
+  echo "gtid_executed: $gtid_executed"
+  echo "gtid_purged:   $gtid_purged"
+  return 1
+fi
+
+local max_pristine_master_tx max_current_master_tx
+max_pristine_master_tx=$(echo "$gtid_executed_pristine" | grep -o -P "${master_server_uuid}:\d+-\d+" | cut -d ":" -f 2 | cut -d "-" -f 2)
+max_current_master_tx=$(echo "$gtid_executed" | grep -o -P "${master_server_uuid}:\d+-\d+" | cut -d ":" -f 2 | cut -d "-" -f 2)
+
+if (( max_current_master_tx <= max_pristine_master_tx )) ; then
+  echo
+  echo "ERROR gtid_executed trimmed legitimate master gtids."
+  echo "current gtid_executed should > prior gtid_executed"
+  echo "prior gtid_executed:   $gtid_executed_pristine"
+  echo "current gtid_executed: $gtid_executed"
+  return 1
+fi

localtests/sql-log-bin-off-replica-gtids/before.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# shellcheck disable=SC2168
+
+local gtid_executed_pristine
+
+# Write something to the replica-only. Make sure it stays even after GTID surgery
+gh-ost-test-mysql-replica test -e "CREATE TABLE replica_only (id INTEGER); INSERT INTO replica_only VALUES (1), (2), (3);"
+
+# shellcheck disable=SC2034
+gtid_executed_pristine=$(gh-ost-test-mysql-replica test -e "SELECT @@global.gtid_executed" -ss)

localtests/sql-log-bin-off-replica-gtids/create.sql

@@ -0,0 +1,26 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  color varchar(32),
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+
+insert into gh_ost_test values (null, 11, 'red');
+insert into gh_ost_test values (null, 13, 'green');
+insert into gh_ost_test values (null, 17, 'blue');
+
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, 'orange');
+  insert into gh_ost_test values (null, 13, 'brown');
+end ;;

localtests/sql-log-bin-off-replica-gtids/extra_args

@@ -0,0 +1 @@
+--throttle-query='select false' --sql-log-bin=false

localtests/sql-log-bin-off-replica-gtids/ignore_versions

@@ -0,0 +1 @@
+(5.5)

localtests/sql-log-bin-off-v5/create.sql

@@ -0,0 +1,26 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  color varchar(32),
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+
+insert into gh_ost_test values (null, 11, 'red');
+insert into gh_ost_test values (null, 13, 'green');
+insert into gh_ost_test values (null, 17, 'blue');
+
+delimiter ;;
+create event gh_ost_test
+  on schedule every 2 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, 'orange');
+  insert into gh_ost_test values (null, 13, 'brown');
+end ;;