Make command default executable saved as list in database

genie-common/src/main/java/com/netflix/genie/common/dto/v4/Command.java

@@ -50,7 +50,7 @@ public class Command extends CommonResource {
     @NotEmpty(message = "At least one executable entry is required")
     private final ImmutableList<
         @NotEmpty(message = "A default executable element shouldn't be an empty string")
-        @Size(max = 255, message = "Executable elements can only be 255 characters") String> executable;
+        @Size(max = 1024, message = "Executable elements can only be 1024 characters") String> executable;
     @Min(
         value = 1,
         message = "The minimum amount of memory if desired is 1 MB. Probably should be much more than that"

genie-web/src/main/java/com/netflix/genie/web/controllers/DtoConverters.java

@@ -305,7 +305,7 @@ public static Command toV4Command(final com.netflix.genie.common.dto.Command v3C
             v3Command.getUpdated().orElse(Instant.now()),
             resources,
             metadataBuilder.build(),
-            Lists.newArrayList(StringUtils.split(v3Command.getExecutable(), ' ')),
+            Lists.newArrayList(StringUtils.split(v3Command.getExecutable())),
             v3Command.getMemory().orElse(null),
             v3Command.getCheckDelay()
         );

genie-web/src/main/java/com/netflix/genie/web/jpa/entities/CommandEntity.java

@@ -26,7 +26,9 @@
 
 import javax.annotation.Nullable;
 import javax.persistence.Basic;
+import javax.persistence.CollectionTable;
 import javax.persistence.Column;
+import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
@@ -38,6 +40,7 @@
 import javax.persistence.Table;
 import javax.validation.constraints.Min;
 import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotEmpty;
 import javax.validation.constraints.NotNull;
 import javax.validation.constraints.Size;
 import java.util.ArrayList;
@@ -67,11 +70,17 @@ public class CommandEntity extends BaseEntity {
     @NotNull(message = "No command status entered and is required.")
     private CommandStatus status;
 
-    @Basic(optional = false)
-    @Column(name = "executable", nullable = false)
-    @NotBlank(message = "No executable entered for command and is required.")
-    @Size(max = 255, message = "Max length in database is 255 characters")
-    private String executable;
+    @ElementCollection
+    @CollectionTable(
+        name = "command_executable_arguments",
+        joinColumns = {
+            @JoinColumn(name = "command_id", nullable = false)
+        }
+    )
+    @Column(name = "argument", length = 1024, nullable = false)
+    @OrderColumn(name = "argument_order", nullable = false)
+    @NotEmpty(message = "No executable arguments entered. At least one is required.")
+    private List<@NotBlank @Size(max = 1024) String> executable = new ArrayList<>();
 
     @Basic(optional = false)
     @Column(name = "check_delay", nullable = false)
@@ -142,6 +151,16 @@ public CommandEntity() {
         super();
     }
 
+    /**
+     * Set the executable and any default arguments for this command.
+     *
+     * @param executable The executable and default arguments which can't be blank and there must be at least one
+     */
+    public void setExecutable(@NotEmpty final List<@NotBlank @Size(max = 1024) String> executable) {
+        this.executable.clear();
+        this.executable.addAll(executable);
+    }
+
     /**
      * Set all the files associated as configuration files for this cluster.
      *

genie-web/src/main/java/com/netflix/genie/web/jpa/entities/JobEntity.java

@@ -51,6 +51,7 @@
 import javax.persistence.Table;
 import javax.validation.constraints.Email;
 import javax.validation.constraints.Min;
+import javax.validation.constraints.NotBlank;
 import javax.validation.constraints.NotNull;
 import javax.validation.constraints.Size;
 import java.time.Instant;
@@ -247,9 +248,9 @@ public class JobEntity extends BaseEntity implements
             @JoinColumn(name = "job_id", nullable = false, updatable = false)
         }
     )
-    @Column(name = "argument", length = 2048, nullable = false, updatable = false)
+    @Column(name = "argument", length = 10_000, nullable = false, updatable = false)
     @OrderColumn(name = "argument_order", nullable = false, updatable = false)
-    private List<String> commandArgs = new ArrayList<>();
+    private List<@NotBlank @Size(max = 10_000) String> commandArgs = new ArrayList<>();
 
     @ManyToMany(fetch = FetchType.LAZY)
     @JoinTable(

genie-web/src/main/java/com/netflix/genie/web/jpa/entities/v4/EntityDtoConverters.java

@@ -20,7 +20,6 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
 import com.netflix.genie.common.dto.v4.Application;
 import com.netflix.genie.common.dto.v4.ApplicationMetadata;
 import com.netflix.genie.common.dto.v4.Cluster;
@@ -38,7 +37,6 @@
 import com.netflix.genie.web.jpa.entities.TagEntity;
 import com.netflix.genie.web.jpa.entities.projections.BaseProjection;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
 
 import java.util.stream.Collectors;
 
@@ -145,7 +143,7 @@ public static Command toV4CommandDto(final CommandEntity commandEntity) {
                 commandEntity.getSetupFile().isPresent() ? commandEntity.getSetupFile().get().getFile() : null
             ),
             metadataBuilder.build(),
-            Lists.newArrayList(StringUtils.split(commandEntity.getExecutable(), ' ')),
+            commandEntity.getExecutable(),
             commandEntity.getMemory().orElse(null),
             commandEntity.getCheckDelay()
         );

genie-web/src/main/java/com/netflix/genie/web/jpa/services/JpaCommandServiceImpl.java

@@ -54,7 +54,6 @@
 import com.netflix.genie.web.services.FileService;
 import com.netflix.genie.web.services.TagService;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageImpl;
@@ -564,8 +563,7 @@ private CommandEntity createCommandEntity(final CommandRequest request) throws G
         final CommandEntity entity = new CommandEntity();
         entity.setUniqueId(request.getRequestedId().orElse(UUID.randomUUID().toString()));
         entity.setCheckDelay(request.getCheckDelay().orElse(com.netflix.genie.common.dto.Command.DEFAULT_CHECK_DELAY));
-        // TODO: Once entity executable saved as List modify this
-        entity.setExecutable(StringUtils.join(request.getExecutable(), ' '));
+        entity.setExecutable(request.getExecutable());
         request.getMemory().ifPresent(entity::setMemory);
         this.setEntityResources(entity, resources);
         this.setEntityTags(entity, metadata);
@@ -587,8 +585,7 @@ private void updateEntityWithDtoContents(
         this.setEntityCommandMetadata(entity, metadata);
 
         entity.setCheckDelay(dto.getCheckDelay());
-        // TODO: Once entity executable saved as List modify this
-        entity.setExecutable(StringUtils.join(dto.getExecutable(), ' '));
+        entity.setExecutable(dto.getExecutable());
         entity.setMemory(dto.getMemory().orElse(null));
     }
 

genie-web/src/main/java/com/netflix/genie/web/jpa/services/JpaServiceUtils.java

@@ -43,6 +43,7 @@
 import com.netflix.genie.web.jpa.entities.projections.JobProjection;
 import com.netflix.genie.web.jpa.entities.projections.JobRequestProjection;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang.StringUtils;
 
 import java.util.stream.Collectors;
 
@@ -131,7 +132,7 @@ static Command toCommandDto(final CommandEntity commandEntity) {
             commandEntity.getUser(),
             commandEntity.getVersion().orElse(NO_VERSION_SUPPLIED),
             commandEntity.getStatus(),
-            commandEntity.getExecutable(),
+            StringUtils.join(commandEntity.getExecutable(), ' '),
             commandEntity.getCheckDelay()
         )
             .withId(commandEntity.getUniqueId())

genie-web/src/main/resources/db/migration/h2/V4_0_0__Genie_4.sql

@@ -16,6 +16,45 @@
  *
  */
 
+CREATE TABLE `command_executable_arguments` (
+  `command_id`     BIGINT(20)    NOT NULL,
+  `argument`       VARCHAR(1024) NOT NULL,
+  `argument_order` INT(11)       NOT NULL,
+  PRIMARY KEY (`command_id`, `argument_order`),
+  CONSTRAINT `COMMAND_EXECUTABLE_ARGUMENTS_COMMAND_ID_FK` FOREIGN KEY (`command_id`) REFERENCES `commands` (`id`)
+  ON DELETE CASCADE
+);
+
+CREATE INDEX `COMMAND_EXECUTABLE_ARGUMENTS_COMMAND_ID_INDEX`
+  ON `command_executable_arguments` (`command_id`);
+
+DROP ALIAS IF EXISTS SPLIT_COMMAND_EXECUTABLE;
+
+CREATE ALIAS SPLIT_COMMAND_EXECUTABLE AS $$
+import java.sql.Connection;
+import java.sql.ResultSet;
+import org.apache.commons.lang3.StringUtils;
+@CODE
+void splitCommandExecutable(final Connection con) throws Exception {
+	final ResultSet rs = con.createStatement().executeQuery("SELECT `id`, `executable` FROM `commands`;");
+
+	while (rs.next()) {
+	    final long commandId = rs.getLong(1);
+	    final String executable = rs.getString(2);
+      final String[] arguments = StringUtils.split(executable);
+      for (int i = 0; i < arguments.length; i++) {
+          con
+              .createStatement()
+              .executeUpdate("INSERT INTO `command_executable_arguments` VALUES (" + commandId + ", '" + arguments[i] + "', " + i + ");");
+      }
+	}
+}
+$$;
+
+CALL SPLIT_COMMAND_EXECUTABLE();
+
+DROP ALIAS IF EXISTS SPLIT_COMMAND_EXECUTABLE;
+
 ALTER TABLE `applications`
   ALTER COLUMN `version` SET NULL;
 ALTER TABLE `applications`
@@ -30,6 +69,8 @@ ALTER TABLE `commands`
   ALTER COLUMN `version` SET NULL;
 ALTER TABLE `commands`
   ALTER COLUMN `version` SET DEFAULT NULL;
+ALTER TABLE `commands`
+  DROP COLUMN `executable`;
 
 ALTER TABLE `jobs`
   ALTER COLUMN `version` SET NULL;

genie-web/src/main/resources/db/migration/mysql/V4_0_0__Genie_4.sql

@@ -34,14 +34,80 @@ DROP TABLE IF EXISTS
 `clusters_320`,
 `commands_320`;
 
+CREATE TABLE `command_executable_arguments` (
+  `command_id`     BIGINT(20)    NOT NULL,
+  `argument`       VARCHAR(1024) NOT NULL,
+  `argument_order` INT(11)       NOT NULL,
+  PRIMARY KEY (`command_id`, `argument_order`),
+  KEY `COMMAND_EXECUTABLE_ARGUMENTS_COMMAND_ID_INDEX` (`command_id`),
+  CONSTRAINT `COMMAND_EXECUTABLE_ARGUMENTS_COMMAND_ID_FK` FOREIGN KEY (`command_id`) REFERENCES `commands` (`id`)
+    ON DELETE CASCADE
+)
+  ENGINE = InnoDB
+  DEFAULT CHARSET = utf8
+  DEFAULT COLLATE = utf8_bin
+  ROW_FORMAT = DYNAMIC;
+
+DELIMITER $$
+CREATE PROCEDURE GENIE_SPLIT_COMMANDS_330()
+  BEGIN
+    DECLARE `done` INT DEFAULT FALSE;
+    DECLARE `command_id` BIGINT(20);
+    DECLARE `command_executable` VARCHAR(255);
+
+    DECLARE `commands_cursor` CURSOR FOR
+      SELECT
+        `id`,
+        `executable`
+      FROM `commands`;
+    DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+
+    START TRANSACTION;
+    OPEN `commands_cursor`;
+    READ_LOOP: LOOP
+      SET `done` = FALSE;
+
+      FETCH `commands_cursor`
+      INTO `command_id`, `command_executable`;
+
+      IF `done`
+      THEN
+        LEAVE READ_LOOP;
+      END IF;
+
+      SET @argument_order = 0;
+      SET @command_executable_local = `command_executable`;
+      COMMAND_EXECUTABLE_LOOP: WHILE LENGTH(@command_executable_local) > 0 DO
+        SET @argument = SUBSTRING_INDEX(@command_executable_local, ' ', 1);
+        SET @command_executable_local = TRIM(LEADING @argument FROM @command_executable_local);
+        SET @command_executable_local = TRIM(LEADING ' ' FROM @command_executable_local);
+        IF LENGTH(@argument) > 0
+        THEN
+          INSERT INTO `command_executable_arguments`
+          VALUES (`command_id`, @argument, @argument_order);
+          SET @argument_order = @argument_order + 1;
+        END IF;
+      END WHILE COMMAND_EXECUTABLE_LOOP;
+
+    END LOOP READ_LOOP;
+    CLOSE `commands_cursor`;
+    COMMIT;
+  END;
+$$
+DELIMITER ;
+
+CALL GENIE_SPLIT_COMMANDS_330();
+DROP PROCEDURE GENIE_SPLIT_COMMANDS_330;
+
 ALTER TABLE `applications`
   CHANGE `version` `version` VARCHAR(255) DEFAULT NULL;
 
 ALTER TABLE `clusters`
   CHANGE `version` `version` VARCHAR(255) DEFAULT NULL;
 
 ALTER TABLE `commands`
-  CHANGE `version` `version` VARCHAR(255) DEFAULT NULL;
+  CHANGE `version` `version` VARCHAR(255) DEFAULT NULL,
+  DROP COLUMN `executable`;
 
 ALTER TABLE `jobs`
   CHANGE `version` `version` VARCHAR(255) DEFAULT NULL;

genie-web/src/main/resources/db/migration/postgresql/V4_0_0__Genie_4.sql

@@ -34,6 +34,57 @@ applications_320,
 clusters_320,
 commands_320;
 
+CREATE TABLE command_executable_arguments (
+  command_id     BIGINT        NOT NULL,
+  argument       VARCHAR(1024) NOT NULL,
+  argument_order INT           NOT NULL,
+  PRIMARY KEY (command_id, argument_order),
+  CONSTRAINT command_executable_arguments_command_id_fkey FOREIGN KEY (command_id) REFERENCES commands (id)
+  ON DELETE CASCADE
+);
+
+CREATE INDEX command_executable_arguments_command_id_index
+  ON command_executable_arguments (command_id);
+
+CREATE OR REPLACE FUNCTION genie_split_commands_330()
+  RETURNS VOID AS $$
+DECLARE
+  command_record   RECORD;
+  executable_local VARCHAR(255);
+  argument         VARCHAR(1024);
+  argument_order   INT;
+BEGIN
+
+  << COMMANDS_LOOP >>
+  FOR command_record IN
+  SELECT
+    id,
+    executable
+  FROM commands
+  LOOP
+
+    argument_order = 0;
+    executable_local = command_record.executable;
+    << COMMAND_ARGS_LOOP >> WHILE LENGTH(executable_local) > 0 LOOP
+      argument = SPLIT_PART(executable_local, ' ', 1);
+      executable_local = TRIM(LEADING argument FROM executable_local);
+      executable_local = TRIM(LEADING ' ' FROM executable_local);
+      IF LENGTH(argument) > 0
+      THEN
+        INSERT INTO command_executable_arguments
+        VALUES (command_record.id, argument, argument_order);
+        argument_order = argument_order + 1;
+      END IF;
+    END LOOP COMMAND_ARGS_LOOP;
+
+  END LOOP COMMANDS_LOOP;
+END;
+$$
+LANGUAGE plpgsql;
+
+SELECT genie_split_commands_330();
+DROP FUNCTION genie_split_commands_330();
+
 ALTER TABLE applications
   ALTER COLUMN version DROP NOT NULL,
   ALTER COLUMN version SET DEFAULT NULL;
@@ -44,7 +95,8 @@ ALTER TABLE clusters
 
 ALTER TABLE commands
   ALTER COLUMN version DROP NOT NULL,
-  ALTER COLUMN version SET DEFAULT NULL;
+  ALTER COLUMN version SET DEFAULT NULL,
+  DROP COLUMN executable;
 
 ALTER TABLE jobs
   ALTER COLUMN version DROP NOT NULL,

genie-web/src/test/groovy/com/netflix/genie/web/jpa/entities/v4/EntityDtoConvertersSpec.groovy

@@ -25,9 +25,7 @@ import com.netflix.genie.common.dto.CommandStatus
 import com.netflix.genie.common.util.GenieObjectMapper
 import com.netflix.genie.test.suppliers.RandomSuppliers
 import com.netflix.genie.web.jpa.entities.*
-import org.apache.commons.lang3.StringUtils
 import spock.lang.Specification
-
 /**
  * Specifications for {@link EntityDtoConverters}.
  *
@@ -213,7 +211,7 @@ class EntityDtoConvertersSpec extends Specification {
                 UUID.randomUUID().toString(),
                 UUID.randomUUID().toString()
         )
-        entity.setExecutable(StringUtils.join(executable, ' '))
+        entity.setExecutable(executable)
         def checkDelay = 2180234L
         entity.setCheckDelay(checkDelay)
         def memory = 10_241