DB updates for constraints

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project are documented below.
 The format is based on [keep a changelog](http://keepachangelog.com/) and this project uses [semantic versioning](http://semver.org/).
 
 ## [Unreleased]
+### Changed
+- Nakama will now log an error and refuse to start if the schema is outdated.
+- Drop unused leaderboard 'next' and 'previous' fields.
+- A user's 'last online at' field now contains a current UTC milliseconds timestamp if they are currently online.
+- Storage remove operations now ignore records that don't exist.
+- Fields that expect JSON content now allow up to 32kb of data.
 
 ## [1.3.0] - 2017-11-21
 ### Added

cmd/migrate.go

@@ -62,12 +62,12 @@ func MigrationStartupCheck(logger *zap.Logger, db *sql.DB) {
 	}
 	records, err := migrate.GetMigrationRecords(db, dialect)
 	if err != nil {
-		logger.Fatal("Could not get migration records", zap.Error(err))
+		logger.Fatal("Could not get migration records, run `nakama migrate up`", zap.Error(err))
 	}
 
 	diff := len(migrations) - len(records)
 	if diff > 0 {
-		logger.Warn("DB schema outdated, run `nakama migrate up`", zap.Int("migrations", diff))
+		logger.Fatal("DB schema outdated, run `nakama migrate up`", zap.Int("migrations", diff))
 	}
 	if diff < 0 {
 		logger.Warn("DB schema newer, update Nakama", zap.Int64("migrations", int64(math.Abs(float64(diff)))))

main.go

@@ -106,7 +106,7 @@ func main() {
 	trackerService.AddDiffListener(presenceNotifier.HandleDiff)
 	notificationService := server.NewNotificationService(jsonLogger, db, trackerService, messageRouter, config.GetSocial().Notification)
 
-	runtimePool, err := server.NewRuntimePool(jsonLogger, multiLogger, db, config.GetRuntime(), notificationService)
+	runtimePool, err := server.NewRuntimePool(jsonLogger, multiLogger, db, config.GetRuntime(), trackerService, notificationService)
 	if err != nil {
 		multiLogger.Fatal("Failed initializing runtime modules.", zap.Error(err))
 	}

migrations/20170115200001_initial_schema.sql

@@ -37,8 +37,7 @@ CREATE TABLE IF NOT EXISTS users (
     created_at     BIGINT        CHECK (created_at > 0) NOT NULL,
     updated_at     BIGINT        CHECK (updated_at > 0) NOT NULL,
     verified_at    BIGINT        CHECK (verified_at >= 0) DEFAULT 0 NOT NULL,
-    disabled_at    BIGINT        CHECK (disabled_at >= 0) DEFAULT 0 NOT NULL,
-    last_online_at BIGINT        CHECK (last_online_at >= 0) DEFAULT 0 NOT NULL
+    disabled_at    BIGINT        CHECK (disabled_at >= 0) DEFAULT 0 NOT NULL
 );
 
 -- This table should be replaced with an array column in the users table

migrations/20170228205100_leaderboards.sql

@@ -17,16 +17,12 @@
 -- +migrate Up
 CREATE TABLE IF NOT EXISTS leaderboard (
     PRIMARY KEY (id),
-    FOREIGN KEY (next_id) REFERENCES leaderboard(id),
-    FOREIGN KEY (prev_id) REFERENCES leaderboard(id),
     id             BYTEA        NOT NULL,
     authoritative  BOOLEAN      DEFAULT FALSE,
     sort_order     SMALLINT     DEFAULT 1 NOT NULL, -- asc(0), desc(1)
     count          BIGINT       DEFAULT 0 CHECK (count >= 0) NOT NULL,
     reset_schedule VARCHAR(64), -- e.g. cron format: "* * * * * * *"
-    metadata       BYTEA        DEFAULT '{}' CHECK (length(metadata) < 16000) NOT NULL,
-    next_id        BYTEA        DEFAULT NULL::BYTEA CHECK (next_id <> id),
-    prev_id        BYTEA        DEFAULT NULL::BYTEA CHECK (prev_id <> id)
+    metadata       BYTEA        DEFAULT '{}' CHECK (length(metadata) < 16000) NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS leaderboard_record (

migrations/20171212154402_last_online.sql

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2017 The Nakama Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- +migrate Up
+ALTER TABLE IF EXISTS leaderboard DROP CONSTRAINT IF EXISTS check_next_id_id;
+ALTER TABLE IF EXISTS leaderboard DROP CONSTRAINT IF EXISTS fk_next_id_ref_leaderboard;
+ALTER TABLE IF EXISTS leaderboard DROP CONSTRAINT IF EXISTS check_prev_id_id;
+ALTER TABLE IF EXISTS leaderboard DROP CONSTRAINT IF EXISTS fk_prev_id_ref_leaderboard;
+
+ALTER TABLE IF EXISTS leaderboard DROP COLUMN IF EXISTS next_id;
+ALTER TABLE IF EXISTS leaderboard DROP COLUMN IF EXISTS prev_id;
+
+ALTER TABLE IF EXISTS users DROP CONSTRAINT IF EXISTS check_last_online_at;
+
+ALTER TABLE IF EXISTS users DROP COLUMN IF EXISTS last_online_at;
+
+ALTER TABLE IF EXISTS users DROP CONSTRAINT IF EXISTS check_metadata;
+ALTER TABLE IF EXISTS users ADD CONSTRAINT check_metadata CHECK (length(metadata) < 32000);
+
+ALTER TABLE IF EXISTS groups DROP CONSTRAINT IF EXISTS check_metadata;
+ALTER TABLE IF EXISTS groups ADD CONSTRAINT check_metadata CHECK (length(metadata) < 32000);
+
+ALTER TABLE IF EXISTS storage DROP CONSTRAINT IF EXISTS check_value;
+ALTER TABLE IF EXISTS storage ADD CONSTRAINT check_value CHECK (length(value) < 32000);
+
+ALTER TABLE IF EXISTS leaderboard_record DROP CONSTRAINT IF EXISTS check_metadata;
+ALTER TABLE IF EXISTS leaderboard_record ADD CONSTRAINT check_metadata CHECK (length(metadata) < 32000);
+
+ALTER TABLE IF EXISTS notification DROP CONSTRAINT IF EXISTS check_content;
+ALTER TABLE IF EXISTS notification ADD CONSTRAINT check_content CHECK (length(content) < 32000);
+
+-- +migrate Down
+ALTER TABLE IF EXISTS users DROP CONSTRAINT IF EXISTS check_metadata;
+ALTER TABLE IF EXISTS users ADD CONSTRAINT check_metadata CHECK (length(metadata) < 16000);
+
+ALTER TABLE IF EXISTS groups DROP CONSTRAINT IF EXISTS check_metadata;
+ALTER TABLE IF EXISTS groups ADD CONSTRAINT check_metadata CHECK (length(metadata) < 16000);
+
+ALTER TABLE IF EXISTS storage DROP CONSTRAINT IF EXISTS check_value;
+ALTER TABLE IF EXISTS storage ADD CONSTRAINT check_value CHECK (length(value) < 16000);
+
+ALTER TABLE IF EXISTS leaderboard_record DROP CONSTRAINT IF EXISTS check_metadata;
+ALTER TABLE IF EXISTS leaderboard_record ADD CONSTRAINT check_metadata CHECK (length(metadata) < 16000);
+
+ALTER TABLE IF EXISTS notification DROP CONSTRAINT IF EXISTS check_content;
+ALTER TABLE IF EXISTS notification ADD CONSTRAINT check_content CHECK (length(content) < 16000);
+
+ALTER TABLE IF EXISTS leaderboard ADD COLUMN IF NOT EXISTS next_id BYTEA DEFAULT NULL::BYTEA;
+ALTER TABLE IF EXISTS leaderboard ADD COLUMN IF NOT EXISTS prev_id BYTEA DEFAULT NULL::BYTEA;
+
+ALTER TABLE IF EXISTS users ADD COLUMN IF NOT EXISTS last_online_at BIGINT NOT NULL DEFAULT 0;
+
+CREATE INDEX ON leaderboard (next_id);
+CREATE INDEX ON leaderboard (prev_id);
+
+-- FIXME cannot create constraints until newly added columns are committed, but that causes issues with the migration.
+-- -- Commit so we can add constraints on the new columns.
+-- COMMIT;
+--
+-- ALTER TABLE IF EXISTS leaderboard ADD CONSTRAINT fk_next_id_ref_leaderboard FOREIGN KEY (next_id) REFERENCES leaderboard(id);
+-- ALTER TABLE IF EXISTS leaderboard ADD CONSTRAINT check_next_id_id CHECK (next_id <> id);
+-- ALTER TABLE IF EXISTS leaderboard ADD CONSTRAINT fk_prev_id_ref_leaderboard FOREIGN KEY (prev_id) REFERENCES leaderboard(id);
+-- ALTER TABLE IF EXISTS leaderboard ADD CONSTRAINT check_prev_id_id CHECK (prev_id <> id);
+
+-- ALTER TABLE IF EXISTS users ADD CONSTRAINT check_last_online_at CHECK (last_online_at >= 0);

pkg/multicode/client_instance.go

@@ -587,7 +587,7 @@ func (c *ClientInstance) flushReliablePacker(ts int64) {
 	if c.reliablePackerLength > 0 {
 		seq, fragments, fragmentLengths, err := c.reliableController.SendPacket(ts, c.reliablePacker, c.reliablePackerLength, RELIABLE_CHANNEL_ID)
 		if err != nil {
-			c.logger.Debug("error flushing reliable packer")
+			c.logger.Debug("error flushing reliable packer", zap.Error(err))
 			return
 		}
 

server/api.proto

@@ -1258,15 +1258,15 @@ message TStorageRemove {
  * Leaderboard is the core domain type representing a Leaderboard setup in the server.
  */
 message Leaderboard {
+  reserved 7, 8;
+
   string id = 1;
   /// Whether the user can submit records directly via the client or not
   bool authoritative = 2;
   int64 sort = 3;
   int64 count = 4;
   string reset_schedule = 5;
   string metadata = 6;
-  string next_id = 7;
-  string prev_id = 8;
 }
 
 /**

server/core_group.go

@@ -396,13 +396,13 @@ WHERE group_edge.destination_id = $1 AND disabled_at = 0 AND (group_edge.state =
 	return groups, 0, nil
 }
 
-func GroupUsersList(logger *zap.Logger, db *sql.DB, caller string, groupID string) ([]*GroupUser, Error_Code, error) {
+func GroupUsersList(logger *zap.Logger, db *sql.DB, tracker Tracker, caller string, groupID string) ([]*GroupUser, Error_Code, error) {
 	groupLogger := logger.With(zap.String("group_id", groupID))
 
 	query := `
 SELECT u.id, u.handle, u.fullname, u.avatar_url,
 	u.lang, u.location, u.timezone, u.metadata,
-	u.created_at, u.updated_at, u.last_online_at, ge.state
+	u.created_at, u.updated_at, ge.state
 FROM users u, group_edge ge
 WHERE u.id = ge.source_id AND ge.destination_id = $1`
 
@@ -413,6 +413,8 @@ WHERE u.id = ge.source_id AND ge.destination_id = $1`
 	}
 	defer rows.Close()
 
+	// If the user is currently online this will be their 'last online at' value.
+	ts := nowMs()
 	users := make([]*GroupUser, 0)
 
 	for rows.Next() {
@@ -426,29 +428,32 @@ WHERE u.id = ge.source_id AND ge.destination_id = $1`
 		var metadata []byte
 		var createdAt sql.NullInt64
 		var updatedAt sql.NullInt64
-		var lastOnlineAt sql.NullInt64
 		var state sql.NullInt64
 
-		err = rows.Scan(&id, &handle, &fullname, &avatarURL, &lang, &location, &timezone, &metadata, &createdAt, &updatedAt, &lastOnlineAt, &state)
+		err = rows.Scan(&id, &handle, &fullname, &avatarURL, &lang, &location, &timezone, &metadata, &createdAt, &updatedAt, &state)
 		if err != nil {
 			groupLogger.Error("Could not get group users, scan error", zap.Error(err))
 			return nil, RUNTIME_EXCEPTION, errors.New("Could not get group users")
 		}
 
+		user := &User{
+			Id:        id.String,
+			Handle:    handle.String,
+			Fullname:  fullname.String,
+			AvatarUrl: avatarURL.String,
+			Lang:      lang.String,
+			Location:  location.String,
+			Timezone:  timezone.String,
+			Metadata:  string(metadata),
+			CreatedAt: createdAt.Int64,
+			UpdatedAt: updatedAt.Int64,
+		}
+		if len(tracker.ListByTopic("notifications:"+id.String)) != 0 {
+			user.LastOnlineAt = ts
+		}
+
 		users = append(users, &GroupUser{
-			User: &User{
-				Id:           id.String,
-				Handle:       handle.String,
-				Fullname:     fullname.String,
-				AvatarUrl:    avatarURL.String,
-				Lang:         lang.String,
-				Location:     location.String,
-				Timezone:     timezone.String,
-				Metadata:     string(metadata),
-				CreatedAt:    createdAt.Int64,
-				UpdatedAt:    updatedAt.Int64,
-				LastOnlineAt: lastOnlineAt.Int64,
-			},
+			User:  user,
 			State: state.Int64,
 		})
 	}