[HWORKS-135] Models backend should store metadata in tables instead o…

…f opensearch (logicalclocks#1023)
ErmiasG · Dec 18, 2023 · 9325e26 · 9325e26
1 parent d6b8661
commit 9325e26
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 4 deletions.
diff --git a/files/default/sql/ddl/3.7.0__initial_tables.sql b/files/default/sql/ddl/3.7.0__initial_tables.sql
@@ -2347,4 +2347,31 @@ CREATE TABLE IF NOT EXISTS `embedding_feature` (
     PRIMARY KEY (`id`),
     KEY `embedding_id` (`embedding_id`),
     CONSTRAINT `embedding_feature_fk` FOREIGN KEY (`embedding_id`) REFERENCES `embedding` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
-    ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+    ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+CREATE TABLE IF NOT EXISTS `model` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` VARCHAR(255) NOT NULL,
+  `project_id` int(11) NOT NULL,
+  UNIQUE KEY `project_unique_name` (`name`, `project_id`),
+  PRIMARY KEY (`id`),
+  CONSTRAINT `model_project_fk` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
+) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+CREATE TABLE IF NOT EXISTS `model_version` (
+  `model_id` int(11) NOT NULL,
+  `version` int(11) NOT NULL,
+  `user_id` int(11) NOT NULL,
+  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `description` VARCHAR(1000) DEFAULT NULL,
+  `metrics` VARCHAR(3000) DEFAULT NULL,
+  `program` VARCHAR(1000) DEFAULT NULL,
+  `framework` VARCHAR(128) DEFAULT NULL,
+  `environment` VARCHAR(1000) DEFAULT NULL,
+  `experiment_id` VARCHAR(128) DEFAULT NULL,
+  `experiment_project_name` VARCHAR(128) DEFAULT NULL,
+  PRIMARY KEY (`model_id`, `version`),
+  CONSTRAINT `user_fk` FOREIGN KEY (`user_id`) REFERENCES `users` (`uid`) ON DELETE NO ACTION ON UPDATE NO ACTION,
+  CONSTRAINT `model_fk` FOREIGN KEY (`model_id`) REFERENCES `model` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
+) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
diff --git a/files/default/sql/ddl/updates/3.7.0.sql b/files/default/sql/ddl/updates/3.7.0.sql
@@ -1,5 +1,5 @@
 -- FSTORE-1047
-CREATE TABLE IF NOT EXISTS `embedding` (
+CREATE TABLE IF NOT EXISTS `hopsworks`.`embedding` (
     `id` int(11) NOT NULL AUTO_INCREMENT,
     `feature_group_id` int(11) NOT NULL,
     `col_prefix` varchar(255) NULL,
@@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS `embedding` (
     CONSTRAINT `feature_group_embedding_fk` FOREIGN KEY (`feature_group_id`) REFERENCES `feature_group` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
     ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
 
-CREATE TABLE IF NOT EXISTS `embedding_feature` (
+CREATE TABLE IF NOT EXISTS `hopsworks`.`embedding_feature` (
     `id` int(11) NOT NULL AUTO_INCREMENT,
     `embedding_id` int(11) NOT NULL,
     `name` varchar(255) NOT NULL,
@@ -18,4 +18,30 @@ CREATE TABLE IF NOT EXISTS `embedding_feature` (
     PRIMARY KEY (`id`),
     KEY `embedding_id` (`embedding_id`),
     CONSTRAINT `embedding_feature_fk` FOREIGN KEY (`embedding_id`) REFERENCES `embedding` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
-    ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+    ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+CREATE TABLE IF NOT EXISTS `hopsworks`.`model` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `name` VARCHAR(255) NOT NULL,
+  `project_id` int(11) NOT NULL,
+  UNIQUE KEY `project_unique_name` (`name`, `project_id`),
+  PRIMARY KEY (`id`),
+  CONSTRAINT `model_project_fk` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
+) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+CREATE TABLE IF NOT EXISTS `hopsworks`.`model_version` (
+  `model_id` int(11) NOT NULL,
+  `version` int(11) NOT NULL,
+  `user_id` int(11) NOT NULL,
+  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `description` VARCHAR(1000) DEFAULT NULL,
+  `metrics` VARCHAR(3000) DEFAULT NULL,
+  `program` VARCHAR(1000) DEFAULT NULL,
+  `framework` VARCHAR(128) DEFAULT NULL,
+  `environment` VARCHAR(1000) DEFAULT NULL,
+  `experiment_id` VARCHAR(128) DEFAULT NULL,
+  `experiment_project_name` VARCHAR(128) DEFAULT NULL,
+  PRIMARY KEY (`model_id`, `version`),
+  CONSTRAINT `user_fk` FOREIGN KEY (`user_id`) REFERENCES `users` (`uid`) ON DELETE NO ACTION ON UPDATE NO ACTION,
+  CONSTRAINT `model_fk` FOREIGN KEY (`model_id`) REFERENCES `model` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION
+) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
diff --git a/templates/default/expat-site.xml.erb b/templates/default/expat-site.xml.erb
@@ -160,4 +160,7 @@
     <version-35>
         io.hops.hopsworks.expat.migrations.airflow.DagsMigration
     </version-35>
+    <version-37>
+        io.hops.hopsworks.expat.migrations.models.OpenSearchToRonDBMigration
+    </version-37>
 </expat>