[HOPSWORKS-2085] redshift connector (logicalclocks#534)

ErmiasG · Dec 5, 2020 · 9ccfe6e · 9ccfe6e
1 parent 35a6227
commit 9ccfe6e
Show file tree

Hide file tree

Showing 3 changed files with 89 additions and 6 deletions.
diff --git a/files/default/sql/ddl/2.1.0__initial_tables.sql b/files/default/sql/ddl/2.1.0__initial_tables.sql
@@ -1837,6 +1837,7 @@ CREATE TABLE IF NOT EXISTS `feature_store_jdbc_connector` (
   `description`             VARCHAR(1000)    NULL,
   `name`                    VARCHAR(1000)    NOT NULL,
   PRIMARY KEY (`id`),
+  UNIQUE KEY `jdbc_connector_feature_store_id_name` (`name`,`feature_store_id`),
   CONSTRAINT `jdbc_connector_featurestore_fk` FOREIGN KEY (`feature_store_id`) REFERENCES `hopsworks`.`feature_store` (`id`)
     ON DELETE CASCADE
     ON UPDATE NO ACTION
@@ -1849,16 +1850,22 @@ CREATE TABLE IF NOT EXISTS `feature_store_s3_connector` (
   `id`                                  INT(11)         NOT NULL AUTO_INCREMENT,
   `feature_store_id`                    INT(11)         NOT NULL,
   `bucket`                              VARCHAR(5000)   NOT NULL,
+  `iam_role`                            VARCHAR(2048)   DEFAULT NULL,
   `description`                         VARCHAR(1000)   NULL,
-  `name`                                VARCHAR(150)   NOT NULL,
+  `name`                                VARCHAR(150)    NOT NULL,
   `server_encryption_algorithm`         INT(11)         NULL,
   `server_encryption_key`               VARCHAR(1000)   NULL,
+  `key_secret_uid`                      INT             DEFAULT NULL,
+  `key_secret_name`                     VARCHAR(200)    DEFAULT NULL,
   PRIMARY KEY (`id`),
+  UNIQUE KEY `s3_connector_feature_store_id_name` (`name`,`feature_store_id`),
+  KEY `fk_feature_store_s3_connector_1_idx` (`key_secret_uid`, `key_secret_name`),
   CONSTRAINT `s3_connector_featurestore_fk` FOREIGN KEY (`feature_store_id`) REFERENCES `hopsworks`.`feature_store` (`id`)
     ON DELETE CASCADE
-    ON UPDATE NO ACTION
-)
-  ENGINE = ndbcluster
+    ON UPDATE NO ACTION,
+  CONSTRAINT `fk_feature_store_s3_connector_1` FOREIGN KEY (`key_secret_uid` , `key_secret_name`) 
+  REFERENCES `hopsworks`.`secrets` (`uid` , `secret_name`) ON DELETE RESTRICT
+) ENGINE = ndbcluster
   DEFAULT CHARSET = latin1
   COLLATE = latin1_general_cs;
 
@@ -2049,3 +2056,30 @@ CREATE TABLE `databricks_instance` (
   FOREIGN KEY `db_secret_fk` (`uid`, `secret_name`) REFERENCES `secrets` (`uid`, `secret_name`) ON DELETE CASCADE ON UPDATE NO ACTION,
   FOREIGN KEY `db_user_fk` (`uid`) REFERENCES `users` (`uid`) ON DELETE CASCADE ON UPDATE NO ACTION
 ) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+CREATE TABLE `feature_store_redshift_connector` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `feature_store_id` int NOT NULL,
+  `cluster_identifier` varchar(64) NOT NULL,
+  `database_driver` varchar(64) NOT NULL,
+  `database_endpoint` varchar(128) DEFAULT NULL,
+  `database_name` varchar(64) DEFAULT NULL,
+  `database_port` int DEFAULT NULL,
+  `table_name` varchar(128) DEFAULT NULL,
+  `database_user_name` varchar(128) DEFAULT NULL,
+  `auto_create` tinyint(1) DEFAULT 0,
+  `database_group` varchar(2048) DEFAULT NULL,
+  `iam_role` varchar(2048) DEFAULT NULL,
+  `arguments` varchar(2000) DEFAULT NULL,
+  `description` varchar(1000) DEFAULT NULL,
+  `name` varchar(150) NOT NULL,
+  `database_pwd_secret_uid` int DEFAULT NULL,
+  `database_pwd_secret_name` varchar(200) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `redshift_connector_feature_store_id_name` (`name`,`feature_store_id`),
+  KEY `fk_feature_store_redshift_connector_1_idx` (`feature_store_id`),
+  KEY `fk_feature_store_redshift_connector_2_idx` (`database_pwd_secret_uid`,`database_pwd_secret_name`),
+  CONSTRAINT `fk_feature_store_redshift_connector_1` FOREIGN KEY (`feature_store_id`) REFERENCES `feature_store` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION,
+  CONSTRAINT `fk_feature_store_redshift_connector_2` FOREIGN KEY (`database_pwd_secret_uid`, `database_pwd_secret_name`)
+  REFERENCES `hopsworks`.`secrets` (`uid`, `secret_name`) ON DELETE RESTRICT
+) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
diff --git a/files/default/sql/ddl/updates/2.1.0.sql b/files/default/sql/ddl/updates/2.1.0.sql
@@ -7,4 +7,44 @@ PREPARE stmt1 FROM @s;
 EXECUTE stmt1;
 DEALLOCATE PREPARE stmt1;
 
-ALTER TABLE `hopsworks`.`conda_commands` CHANGE `environment_yml` `environment_file` VARCHAR(1000) COLLATE latin1_general_cs DEFAULT NULL;
+ALTER TABLE `hopsworks`.`conda_commands` CHANGE `environment_yml` `environment_file` VARCHAR(1000) COLLATE latin1_general_cs DEFAULT NULL;
+ALTER TABLE `hopsworks`.`feature_store_jdbc_connector` ADD UNIQUE INDEX `jdbc_connector_feature_store_id_name` (`feature_store_id`, `name`);
+
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` ADD UNIQUE INDEX `s3_connector_feature_store_id_name` (`feature_store_id`, `name`);
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` ADD COLUMN `iam_role` VARCHAR(2048) DEFAULT NULL;
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` ADD COLUMN `key_secret_uid` INT DEFAULT NULL;
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` ADD COLUMN `key_secret_name` VARCHAR(200) DEFAULT NULL;
+
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` 
+ADD INDEX `fk_feature_store_s3_connector_1_idx` (`key_secret_uid`, `key_secret_name`);
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` ADD CONSTRAINT `fk_feature_store_s3_connector_1`
+  FOREIGN KEY (`key_secret_uid` , `key_secret_name`)
+  REFERENCES `hopsworks`.`secrets` (`uid` , `secret_name`)
+  ON DELETE RESTRICT;
+
+CREATE TABLE `feature_store_redshift_connector` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `feature_store_id` int NOT NULL,
+  `cluster_identifier` varchar(64) NOT NULL,
+  `database_driver` varchar(64) NOT NULL,
+  `database_endpoint` varchar(128) DEFAULT NULL,
+  `database_name` varchar(64) DEFAULT NULL,
+  `database_port` int DEFAULT NULL,
+  `table_name` varchar(128) DEFAULT NULL,
+  `database_user_name` varchar(128) DEFAULT NULL,
+  `auto_create` tinyint(1) DEFAULT 0,
+  `database_group` varchar(2048) DEFAULT NULL,
+  `iam_role` varchar(2048) DEFAULT NULL,
+  `arguments` varchar(2000) DEFAULT NULL,
+  `description` varchar(1000) DEFAULT NULL,
+  `name` varchar(150) NOT NULL,
+  `database_pwd_secret_uid` int DEFAULT NULL,
+  `database_pwd_secret_name` varchar(200) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `redshift_connector_feature_store_id_name` (`name`,`feature_store_id`),
+  KEY `fk_feature_store_redshift_connector_1_idx` (`feature_store_id`),
+  KEY `fk_feature_store_redshift_connector_2_idx` (`database_pwd_secret_uid`,`database_pwd_secret_name`),
+  CONSTRAINT `fk_feature_store_redshift_connector_1` FOREIGN KEY (`feature_store_id`) REFERENCES `feature_store` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION,
+  CONSTRAINT `fk_feature_store_redshift_connector_2` FOREIGN KEY (`database_pwd_secret_uid`, `database_pwd_secret_name`)
+  REFERENCES `hopsworks`.`secrets` (`uid`, `secret_name`) ON DELETE RESTRICT
+) ENGINE=ndbcluster DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
diff --git a/files/default/sql/ddl/updates/undo/2.1.0__undo.sql b/files/default/sql/ddl/updates/undo/2.1.0__undo.sql
@@ -1,3 +1,12 @@
 ALTER TABLE `hopsworks`.`conda_commands` ADD CONSTRAINT `FK_284_520` FOREIGN KEY (`project_id`) REFERENCES `project` (`id`) ON DELETE CASCADE ON UPDATE NO ACTION;
 
-ALTER TABLE `hopsworks`.`conda_commands` CHANGE `environment_file` `environment_yml` VARCHAR(1000) COLLATE latin1_general_cs DEFAULT NULL;
+ALTER TABLE `hopsworks`.`conda_commands` CHANGE `environment_file` `environment_yml` VARCHAR(1000) COLLATE latin1_general_cs DEFAULT NULL;
+ALTER TABLE `hopsworks`.`feature_store_jdbc_connector` DROP INDEX `jdbc_connector_feature_store_id_name`;
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` DROP INDEX `s3_connector_feature_store_id_name`;
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` DROP INDEX `fk_feature_store_s3_connector_1_idx`;
+
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` DROP COLUMN `iam_role`;
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` DROP COLUMN `key_secret_uid`;
+ALTER TABLE `hopsworks`.`feature_store_s3_connector` DROP COLUMN `key_secret_name`;
+
+DROP TABLE IF EXISTS `hopsworks`.`feature_store_redshift_connector`;