Rename and add access_token

* Add `elasticsearch.access_token` setting
* Rename `workplace_search.access_token` setting. It defaults to `elasticsearch.access_token`.
* Add `--access_token` CLI option

3rdparty/workplacesearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/thirdparty/wpsearch/WPSearchClient.java

@@ -77,7 +77,7 @@ public class WPSearchClient implements Closeable {
     private String username;
     @Deprecated
     private String password;
-    private String elasticsearchToken;
+    private String accessToken;
     private int bulkSize;
     private TimeValue flushInterval;
 
@@ -87,7 +87,7 @@ public class WPSearchClient implements Closeable {
     private final Path rootDir;
     private final Path jobMappingDir;
     private String version;
-    private String elasticsearchTokenHeader = null;
+    private String authorizationHeader = null;
 
     /**
      * Create a client
@@ -102,7 +102,7 @@ public WPSearchClient(Path rootDir, Path jobMappingDir) {
      * @param username Username
      * @param defaultValue default value to use for username. Defaults to Elasticsearch username.
      * @return the current instance
-     * @deprecated use {@link #withElasticsearchToken(String)} instead
+     * @deprecated use {@link #withAccessToken(String)} instead
      */
     @Deprecated
     public WPSearchClient withUsername(String username, String defaultValue) {
@@ -119,7 +119,7 @@ public WPSearchClient withUsername(String username, String defaultValue) {
      * @param password Password
      * @param defaultValue default value to use for password. Defaults to Elasticsearch password.
      * @return the current instance
-     * @deprecated use {@link #withElasticsearchToken(String)} instead
+     * @deprecated use {@link #withAccessToken(String)} instead
      */
     @Deprecated
     public WPSearchClient withPassword(String password, String defaultValue) {
@@ -132,13 +132,13 @@ public WPSearchClient withPassword(String password, String defaultValue) {
     }
 
     /**
-     * Defines the Elasticsearch Token. Defaults to null.
+     * Defines the Access Token. Defaults to null.
      * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/8.12/security-api-get-token.html">Get Token API</a>
-     * @param  elasticsearchToken  The Elasticsearch Token
+     * @param  accessToken  The Access Token
      * @return the current instance
      */
-    public WPSearchClient withElasticsearchToken(String elasticsearchToken) {
-        this.elasticsearchToken = elasticsearchToken;
+    public WPSearchClient withAccessToken(String accessToken) {
+        this.accessToken = accessToken;
         return this;
     }
 
@@ -193,11 +193,11 @@ public void start() {
                 .withConfig(config);
 
         // If we have an ApiKey, let's use it. Otherwise, we will use basic auth
-        if (FsCrawlerUtil.isNullOrEmpty(elasticsearchToken)) {
+        if (FsCrawlerUtil.isNullOrEmpty(accessToken)) {
             HttpAuthenticationFeature feature = HttpAuthenticationFeature.basic(username, password);
             clientBuilder.register(feature);
         } else {
-            elasticsearchTokenHeader = "Bearer " + elasticsearchToken;
+            authorizationHeader = "Bearer " + accessToken;
         }
 
         client = clientBuilder.build();
@@ -526,8 +526,8 @@ private Invocation.Builder prepareHttpCall(String urlForApi, String path) {
                 .request(MediaType.APPLICATION_JSON)
                 .header(HttpHeaders.CONTENT_TYPE, "application/json");
         builder.header(HttpHeaders.USER_AGENT, USER_AGENT);
-        if (elasticsearchTokenHeader != null) {
-            builder.header(HttpHeaders.AUTHORIZATION, elasticsearchTokenHeader);
+        if (authorizationHeader != null) {
+            builder.header(HttpHeaders.AUTHORIZATION, authorizationHeader);
         }
 
         return builder;
@@ -536,7 +536,7 @@ private Invocation.Builder prepareHttpCall(String urlForApi, String path) {
     @Override
     public String toString() {
         return "WPSearchClient{" +
-                ", host='" + host + '\'' +
+                " host='" + host + '\'' +
                 '}';
     }
 }

cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java

@@ -78,7 +78,10 @@ public static class FsCrawlerCommand {
         @Parameter(names = "--api_key", description = "Elasticsearch api key. See https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html")
         String apiKey = null;
 
-        @Parameter(names = "--username", description = "Elasticsearch username when running with security.  (Deprecated - use --api_key instead)")
+        @Parameter(names = "--access_token", description = "Elasticsearch access token. See https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-get-token.html")
+        String accessToken = null;
+
+        @Parameter(names = "--username", description = "Elasticsearch username. (Deprecated - use --api_key or --access_token instead)")
         @Deprecated
         String username = null;
 
@@ -222,11 +225,12 @@ static FsSettings loadSettings(Path configDir, String jobName) throws IOExceptio
     /**
      * Modify existing settings with correct default values when not set.
      *
-     * @param fsSettings    the settings to modify
-     * @param usernameCli   the username coming from the CLI if any (deprecated)
-     * @param apiKeyCli     the api key coming from the CLI if any
+     * @param fsSettings        the settings to modify
+     * @param usernameCli       the username coming from the CLI if any (deprecated)
+     * @param apiKeyCli         the api key coming from the CLI if any
+     * @param accessTokenCli    the access token coming from the CLI if any
      */
-    static void modifySettings(FsSettings fsSettings, String usernameCli, String apiKeyCli) {
+    static void modifySettings(FsSettings fsSettings, String usernameCli, String apiKeyCli, String accessTokenCli) {
         // Check default settings
         if (fsSettings.getFs() == null) {
             fsSettings.setFs(Fs.DEFAULT);
@@ -257,6 +261,11 @@ static void modifySettings(FsSettings fsSettings, String usernameCli, String api
         if (fsSettings.getElasticsearch().getApiKey() == null && apiKeyCli != null) {
             fsSettings.getElasticsearch().setApiKey(apiKeyCli);
         }
+
+        // Overwrite settings with command line values
+        if (fsSettings.getElasticsearch().getAccessToken() == null && accessTokenCli != null) {
+            fsSettings.getElasticsearch().setAccessToken(accessTokenCli);
+        }
     }
 
     /**
@@ -373,7 +382,7 @@ static void runner(FsCrawlerCommand command) throws IOException {
             return;
         }
 
-        modifySettings(fsSettings, command.username, command.apiKey);
+        modifySettings(fsSettings, command.username, command.apiKey, command.accessToken);
         if (fsSettings.getElasticsearch().getUsername() != null && fsSettings.getElasticsearch().getPassword() == null && scanner != null) {
             FSCrawlerLogger.console("Password for {}:", fsSettings.getElasticsearch().getUsername());
             String password = scanner.next();

cli/src/test/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCliDefaultSettingsTest.java

@@ -83,11 +83,12 @@ public void testModifySettingsNoUsername() throws IOException {
         FsSettings settings = fsSettingsFileHandler.read("modify_settings_no_username");
         assertThat(settings.getFs(), nullValue());
         assertThat(settings.getElasticsearch(), nullValue());
-        modifySettings(settings, null, null);
+        modifySettings(settings, null, null, null);
         assertThat(settings.getFs(), notNullValue());
         assertThat(settings.getElasticsearch(), notNullValue());
         assertThat(settings.getElasticsearch().getUsername(), nullValue());
         assertThat(settings.getElasticsearch().getApiKey(), nullValue());
+        assertThat(settings.getElasticsearch().getAccessToken(), nullValue());
     }
 
     @Test
@@ -100,11 +101,12 @@ public void testModifySettingsWithUsernameDeprecated() throws IOException {
         FsSettings settings = fsSettingsFileHandler.read("modify_settings_with_username");
         assertThat(settings.getFs(), nullValue());
         assertThat(settings.getElasticsearch(), nullValue());
-        modifySettings(settings, "elastic", null);
+        modifySettings(settings, "elastic", null, null);
         assertThat(settings.getFs(), notNullValue());
         assertThat(settings.getElasticsearch(), notNullValue());
         assertThat(settings.getElasticsearch().getUsername(), is("elastic"));
         assertThat(settings.getElasticsearch().getApiKey(), nullValue());
+        assertThat(settings.getElasticsearch().getAccessToken(), nullValue());
     }
 
     @Test
@@ -117,11 +119,30 @@ public void testModifySettingsWithApiKey() throws IOException {
         FsSettings settings = fsSettingsFileHandler.read("modify_settings_with_username");
         assertThat(settings.getFs(), nullValue());
         assertThat(settings.getElasticsearch(), nullValue());
-        modifySettings(settings, null, "my_api_key_base64_encoded");
+        modifySettings(settings, null, "my_api_key_base64_encoded", null);
         assertThat(settings.getFs(), notNullValue());
         assertThat(settings.getElasticsearch(), notNullValue());
         assertThat(settings.getElasticsearch().getUsername(), nullValue());
         assertThat(settings.getElasticsearch().getApiKey(), is("my_api_key_base64_encoded"));
+        assertThat(settings.getElasticsearch().getAccessToken(), nullValue());
+    }
+
+    @Test
+    public void testModifySettingsWithAccessToken() throws IOException {
+        FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir);
+        Path jobDir = metadataDir.resolve("modify_settings_with_username");
+        Files.createDirectories(jobDir);
+
+        Files.writeString(jobDir.resolve(SETTINGS_YAML), "name: \"modify_settings_with_username\"");
+        FsSettings settings = fsSettingsFileHandler.read("modify_settings_with_username");
+        assertThat(settings.getFs(), nullValue());
+        assertThat(settings.getElasticsearch(), nullValue());
+        modifySettings(settings, null, null, "my_access_token");
+        assertThat(settings.getFs(), notNullValue());
+        assertThat(settings.getElasticsearch(), notNullValue());
+        assertThat(settings.getElasticsearch().getUsername(), nullValue());
+        assertThat(settings.getElasticsearch().getApiKey(), nullValue());
+        assertThat(settings.getElasticsearch().getAccessToken(), is("my_access_token"));
     }
 
     @Test
@@ -141,11 +162,12 @@ public void testModifySettingsWithServerFtp() throws IOException {
         assertThat(settings.getServer(), notNullValue());
         assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT));
         assertThat(settings.getServer().getUsername(), nullValue());
-        modifySettings(settings, null, "my_api_key_base64_encoded");
+        modifySettings(settings, null, "my_api_key_base64_encoded", null);
         assertThat(settings.getFs(), notNullValue());
         assertThat(settings.getElasticsearch(), notNullValue());
         assertThat(settings.getElasticsearch().getUsername(), nullValue());
         assertThat(settings.getElasticsearch().getApiKey(), is("my_api_key_base64_encoded"));
+        assertThat(settings.getElasticsearch().getAccessToken(), nullValue());
         assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.FTP_PORT));
         assertThat(settings.getServer().getUsername(), is("anonymous"));
     }
@@ -167,11 +189,12 @@ public void testModifySettingsWithServerSsh() throws IOException {
         assertThat(settings.getServer(), notNullValue());
         assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT));
         assertThat(settings.getServer().getUsername(), nullValue());
-        modifySettings(settings, null, "my_api_key_base64_encoded");
+        modifySettings(settings, null, "my_api_key_base64_encoded", null);
         assertThat(settings.getFs(), notNullValue());
         assertThat(settings.getElasticsearch(), notNullValue());
         assertThat(settings.getElasticsearch().getUsername(), nullValue());
         assertThat(settings.getElasticsearch().getApiKey(), is("my_api_key_base64_encoded"));
+        assertThat(settings.getElasticsearch().getAccessToken(), nullValue());
         assertThat(settings.getServer().getPort(), is(Server.PROTOCOL.SSH_PORT));
         assertThat(settings.getServer().getUsername(), nullValue());
     }

elasticsearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClient.java

@@ -96,7 +96,7 @@ public class ElasticsearchClient implements IElasticsearchClient {
     private int majorVersion;
     private int currentNode = -1;
     private int currentRun = -1;
-    private String apiKeyHeader = null;
+    private String authorizationHeader = null;
 
     public ElasticsearchClient(Path config, FsSettings settings) {
         this.config = config;
@@ -185,14 +185,16 @@ public void start() throws ElasticsearchClientException {
         ClientBuilder clientBuilder = ClientBuilder.newBuilder()
                 .withConfig(config);
 
-        // If we have an ApiKey, let's use it. Otherwise, we will use basic auth
-        if (FsCrawlerUtil.isNullOrEmpty(settings.getElasticsearch().getApiKey())) {
+        // If we have an Api Key or an Access Token, let's use it. Otherwise, we will use basic auth
+        if (!FsCrawlerUtil.isNullOrEmpty(settings.getElasticsearch().getApiKey())) {
+            authorizationHeader = "ApiKey " + settings.getElasticsearch().getApiKey();
+        } else if (!FsCrawlerUtil.isNullOrEmpty(settings.getElasticsearch().getAccessToken())) {
+            authorizationHeader = "Bearer " + settings.getElasticsearch().getAccessToken();
+        } else {
             HttpAuthenticationFeature feature = HttpAuthenticationFeature.basic(
                     settings.getElasticsearch().getUsername(),
                     settings.getElasticsearch().getPassword());
             clientBuilder.register(feature);
-        } else {
-            apiKeyHeader = "ApiKey " + settings.getElasticsearch().getApiKey();
         }
         if (sslContext != null) {
             clientBuilder.sslContext(sslContext);
@@ -1001,8 +1003,8 @@ private Invocation.Builder prepareHttpCall(String node, String path, Map.Entry<S
                 .request(MediaType.APPLICATION_JSON)
                 .header(HttpHeaders.CONTENT_TYPE, "application/json");
         builder.header(HttpHeaders.USER_AGENT, USER_AGENT);
-        if (apiKeyHeader != null) {
-            builder.header(HttpHeaders.AUTHORIZATION, apiKeyHeader);
+        if (authorizationHeader != null) {
+            builder.header(HttpHeaders.AUTHORIZATION, authorizationHeader);
         }
 
         return builder;

elasticsearch-client/src/main/java/fr/pilato/elasticsearch/crawler/fs/client/WorkplaceSearchClient.java

@@ -61,9 +61,12 @@ public void start() throws IOException {
             .withBulkSize(settings.getWorkplaceSearch().getBulkSize())
             .withFlushInterval(settings.getWorkplaceSearch().getFlushInterval());
 
-        if (settings.getWorkplaceSearch().getElasticsearchToken() != null) {
+        if (settings.getWorkplaceSearch().getAccessToken() != null) {
+            // Using Workplace Search Access Token
+            wpSearchClient.withAccessToken(settings.getWorkplaceSearch().getAccessToken());
+        } else if (settings.getElasticsearch().getAccessToken() != null) {
             // Using Elasticsearch Access Token
-            wpSearchClient.withElasticsearchToken(settings.getWorkplaceSearch().getElasticsearchToken());
+            wpSearchClient.withAccessToken(settings.getElasticsearch().getAccessToken());
         } else {
             wpSearchClient
                     .withUsername(settings.getWorkplaceSearch().getUsername(), settings.getElasticsearch().getUsername())

integration-tests/src/test/java/fr/pilato/elasticsearch/crawler/fs/test/integration/AbstractITCase.java

@@ -246,8 +246,13 @@ public static void startServices() throws IOException, ElasticsearchClientExcept
                 .setNodes(Collections.singletonList(new ServerUrl(testClusterUrl)))
                 .setSslVerification(false);
         if (testApiKey != null) {
+            staticLogger.debug("using api key [{}]", testApiKey);
             builder.setApiKey(testApiKey);
+        } else if (testAccessToken != null) {
+            staticLogger.debug("using access token [{}]", testAccessToken);
+            builder.setAccessToken(testAccessToken);
         } else {
+            staticLogger.debug("using login/password [{}]/[{}]", testClusterUser, testClusterPass);
             builder.setUsername(testClusterUser);
             builder.setPassword(testClusterPass);
         }
@@ -257,6 +262,19 @@ public static void startServices() throws IOException, ElasticsearchClientExcept
         documentService = new FsCrawlerDocumentServiceElasticsearchImpl(metadataDir, fsSettings);
         try {
             documentService.start();
+
+            // We make sure the cluster is running
+            managementService = new FsCrawlerManagementServiceElasticsearchImpl(metadataDir, fsSettings);
+            managementService.start();
+
+            // Generate the Api-Key
+            if (testApiKey == null) {
+                testApiKey = managementService.getClient().generateApiKey("fscrawler");
+            }
+
+            if (testAccessToken == null) {
+                testAccessToken = managementService.getClient().generateElasticsearchToken();
+            }
         } catch (ElasticsearchClientException e) {
             if ((e.getCause() instanceof SocketException ||
                     (e.getCause() instanceof ProcessingException && e.getCause().getCause() instanceof SSLException))
@@ -268,32 +286,17 @@ public static void startServices() throws IOException, ElasticsearchClientExcept
                 Elasticsearch.Builder builderForOlderService = Elasticsearch.builder()
                         .setNodes(Collections.singletonList(new ServerUrl(testClusterUrl)))
                         .setSslVerification(false);
-                if (testApiKey != null) {
-                    builderForOlderService.setApiKey(testApiKey);
-                } else {
-                    builderForOlderService.setUsername(testClusterUser);
-                    builderForOlderService.setPassword(testClusterPass);
-                }
+
+                // For older versions, we want to test using login/password
+                builderForOlderService.setUsername(testClusterUser);
+                builderForOlderService.setPassword(testClusterPass);
                 elasticsearchWithSecurity = builderForOlderService.build();
                 fsSettings = FsSettings.builder("esClient").setElasticsearch(elasticsearchWithSecurity).build();
                 documentService.close();
                 documentService = null;
             }
         }
 
-        // If an ApiKey was not provided, we need to generate one and recreate the services
-        if (testApiKey == null) {
-            // We make sure the cluster is running
-            managementService = new FsCrawlerManagementServiceElasticsearchImpl(metadataDir, fsSettings);
-            managementService.start();
-            // Generate the Api-Key
-            testApiKey = managementService.getClient().generateApiKey("fscrawler");
-            managementService.close();
-            managementService = null;
-
-            staticLogger.info("Restarting the services using the generated key [{}]", testApiKey);
-        }
-
         // We make sure the cluster is running
         try {
             if (documentService == null) {
@@ -307,9 +310,6 @@ public static void startServices() throws IOException, ElasticsearchClientExcept
 
             String version = managementService.getVersion();
             staticLogger.info("Starting integration tests against an external cluster running elasticsearch [{}]", version);
-
-            staticLogger.info("Generating an access token to use with Workplace Search");
-            testAccessToken = managementService.getClient().generateElasticsearchToken();
         } catch (ConnectException e) {
             // If we have an exception here, let's ignore the test
             staticLogger.warn("Integration tests are skipped: [{}]", e.getMessage());