Adding option to debug the cli launcher (elastic#102464)

* Adding debug args to cli tools
* Adding instructions

TESTING.asciidoc

@@ -45,7 +45,7 @@ run it using Gradle:
 
 ==== Launching and debugging from an IDE
 
-If you want to run Elasticsearch from your IDE, the `./gradlew run` task
+If you want to run and debug Elasticsearch from your IDE, the `./gradlew run` task
 supports a remote debugging option. Run the following from your terminal:
 
 ---------------------------------------------------------------------------
@@ -55,7 +55,7 @@ supports a remote debugging option. Run the following from your terminal:
 Next start the "Debug Elasticsearch" run configuration in IntelliJ. This will enable the IDE to connect to the process and allow debug functionality.
 
 
-As such the IDE needs to be instructed to listen for connections on this port.
+As such the IDE needs to be instructed to listen for connections on the debug port.
 Since we might run multiple JVMs as part of configuring and starting the cluster it's
 recommended to configure the IDE to initiate multiple listening attempts. In case of IntelliJ, this option
 is called "Auto restart" and needs to be checked.
@@ -64,6 +64,22 @@ NOTE: If you have imported the project into IntelliJ according to the instructio
 link:/CONTRIBUTING.md#importing-the-project-into-intellij-idea[CONTRIBUTING.md] then a debug run configuration
 named "Debug Elasticsearch" will be created for you and configured appropriately.
 
+===== Debugging the CLI launcher
+
+The gradle task does not start the Elasticsearch server process directly; like in the Elasticsearch distribution,
+the job of starting the server process is delegated to a launcher CLI tool. If you need to debug the launcher itself,
+add the following option to the `run` task:
+---------------------------------------------------------------------------
+./gradlew run --debug-cli-jvm
+---------------------------------------------------------------------------
+This option can be specified in isolation or combined with `--debug-jvm`. Since the CLI launcher lifespan may overlap
+with the server process lifespan, the CLI launcher process will be started on a different port (5107 for the first node,
+5108 and following for additional cluster nodes).
+
+As with the `--debug-jvm` command, the IDE needs to be instructed to listen for connections on the debug port.
+You need to configure and start an appropriate Remote JVM Debug configuration, e.g. by cloning and editing
+the "Debug Elasticsearch" run configuration to point to the correct debug port.
+
 ==== Disabling assertions
 
 When running Elasticsearch with `./gradlew run`, assertions are enabled by
@@ -103,7 +119,8 @@ password: `elastic-password`.
 - In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
 - In order to use a custom data directory: `--data-dir=/tmp/foo`
 - In order to preserve data in between executions: `--preserve-data`
-- In order to remotely attach a debugger to the process: `--debug-jvm`
+- In order to remotely attach a debugger to the server process: `--debug-jvm`
+- In order to remotely attach a debugger to the CLI launcher process: `--debug-cli-jvm`
 - In order to set a different keystore password: `--keystore-password`
 - In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
 - In order to pass a JVM setting, e.g. to disable assertions: `-Dtests.jvm.argline="-da"`

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -151,6 +151,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final LazyPropertyMap<String, CharSequence> systemProperties = new LazyPropertyMap<>("System properties", this);
     private final LazyPropertyMap<String, CharSequence> environment = new LazyPropertyMap<>("Environment", this);
     private final LazyPropertyList<CharSequence> jvmArgs = new LazyPropertyList<>("JVM arguments", this);
+    private final LazyPropertyList<CharSequence> cliJvmArgs = new LazyPropertyList<>("CLI JVM arguments", this);
     private final LazyPropertyMap<String, File> extraConfigFiles = new LazyPropertyMap<>("Extra config files", this, FileEntry::new);
     private final LazyPropertyList<FileCollection> extraJarConfigurations = new LazyPropertyList<>("Extra jar files", this);
     private final List<Map<String, String>> credentials = new ArrayList<>();
@@ -471,6 +472,10 @@ public void jvmArgs(String... values) {
         jvmArgs.addAll(Arrays.asList(values));
     }
 
+    public void cliJvmArgs(String... values) {
+        cliJvmArgs.addAll(Arrays.asList(values));
+    }
+
     @Internal
     public Path getConfigDir() {
         return configFile.getParent();
@@ -932,6 +937,10 @@ private void startElasticsearchProcess() {
         // Don't inherit anything from the environment for as that would lack reproducibility
         environment.clear();
         environment.putAll(getESEnvironment());
+        if (cliJvmArgs.isEmpty() == false) {
+            String cliJvmArgsString = String.join(" ", cliJvmArgs);
+            environment.put("CLI_JAVA_OPTS", cliJvmArgsString);
+        }
 
         // Direct the stderr to the ES log file. This should capture any jvm problems to start.
         // Stdout is discarded because ES duplicates the log file to stdout when run in the foreground.

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -40,6 +40,7 @@ public abstract class RunTask extends DefaultTestClustersTask {
     private static final String transportCertificate = "private-cert2.p12";
 
     private Boolean debug = false;
+    private Boolean cliDebug = false;
     private Boolean apmServerEnabled = false;
 
     private Boolean preserveData = false;
@@ -62,11 +63,21 @@ public void setDebug(boolean enabled) {
         this.debug = enabled;
     }
 
+    @Option(option = "debug-cli-jvm", description = "Enable debugging configuration, to allow attaching a debugger to the cli launcher.")
+    public void setCliDebug(boolean enabled) {
+        this.cliDebug = enabled;
+    }
+
     @Input
     public Boolean getDebug() {
         return debug;
     }
 
+    @Input
+    public Boolean getCliDebug() {
+        return cliDebug;
+    }
+
     @Input
     public Boolean getApmServerEnabled() {
         return apmServerEnabled;
@@ -204,6 +215,9 @@ public void beforeStart() {
         if (debug) {
             enableDebug();
         }
+        if (cliDebug) {
+            enableCliDebug();
+        }
     }
 
     @TaskAction

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java

@@ -54,4 +54,19 @@ default void enableDebug() {
             }
         }
     }
+
+    default void enableCliDebug() {
+        int cliDebugPort = 5107;
+        for (ElasticsearchCluster cluster : getClusters()) {
+            for (ElasticsearchNode node : cluster.getNodes()) {
+                getLogger().lifecycle(
+                    "Running cli launcher in debug mode, {} expecting running debug server on port {}",
+                    node,
+                    cliDebugPort
+                );
+                node.cliJvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=" + cliDebugPort);
+                cliDebugPort += 1;
+            }
+        }
+    }
 }