Cleanup code

.gitignore

@@ -1,12 +1,7 @@
-/data
 /target
 /.settings
 /.classpath
 /.project
 .idea
 *.iml
-.local-execution-hints.log
-.local-*-execution-hints.log
 plugin_tools
-config
-logs

src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java

@@ -29,7 +29,6 @@
 import fr.pilato.elasticsearch.crawler.fs.meta.doc.PathParser;
 import fr.pilato.elasticsearch.crawler.fs.meta.job.FsJob;
 import fr.pilato.elasticsearch.crawler.fs.meta.job.FsJobFileHandler;
-import fr.pilato.elasticsearch.crawler.fs.meta.settings.Elasticsearch;
 import fr.pilato.elasticsearch.crawler.fs.meta.settings.FsSettings;
 import fr.pilato.elasticsearch.crawler.fs.meta.settings.FsSettingsFileHandler;
 import fr.pilato.elasticsearch.crawler.fs.util.FsCrawlerUtil;
@@ -126,9 +125,7 @@ public void start() throws Exception {
             // Create an elasticsearch client
             client = ElasticsearchClient.builder().build();
 
-            for (Elasticsearch.Node node : settings.getElasticsearch().getNodes()) {
-                client.addNode(node);
-            }
+            settings.getElasticsearch().getNodes().forEach(client::addNode);
 
             client.createIndex(settings.getElasticsearch().getIndex(), true);
         } catch (Exception e) {
@@ -178,7 +175,7 @@ public boolean isClosed() {
     }
 
     private class FSParser implements Runnable {
-        private FsSettings fsSettings;
+        private final FsSettings fsSettings;
 
         private ScanStatistic stats;
 
@@ -647,7 +644,7 @@ private void esIndex(String index, String type, String id, fr.pilato.elasticsear
         /**
          * Add to bulk an IndexRequest in JSon format
          */
-        private void esIndex(String index, String type, String id, String json) throws Exception {
+        private void esIndex(String index, String type, String id, String json) {
             logger.debug("Indexing in ES " + index + ", " + type + ", " + id);
             logger.trace("JSon indexed : {}", json);
 
@@ -661,7 +658,7 @@ private void esIndex(String index, String type, String id, String json) throws E
         /**
          * Add to bulk a DeleteRequest
          */
-        private void esDelete(String index, String type, String id) throws Exception {
+        private void esDelete(String index, String type, String id) {
             logger.debug("Deleting from ES " + index + ", " + type + ", " + id);
             if (!closed) {
                 bulkProcessor.add(new DeleteRequest(index, type, id));

src/main/java/fr/pilato/elasticsearch/crawler/fs/SignTool.java

@@ -36,8 +36,8 @@ public static String sign(String toSign) throws NoSuchAlgorithmException {
 
         String key = "";
         byte b[] = md.digest();
-        for (int i = 0; i < b.length; i++) {
-            long t = b[i] < 0 ? 256 + b[i] : b[i];
+        for (byte aB : b) {
+            long t = aB < 0 ? 256 + aB : aB;
             key += Long.toHexString(t);
         }
 

src/main/java/fr/pilato/elasticsearch/crawler/fs/client/BulkProcessor.java

@@ -43,7 +43,7 @@ public class BulkProcessor {
     private volatile boolean closed = false;
     private final AtomicLong executionIdGen = new AtomicLong();
 
-    public BulkProcessor(ElasticsearchClient client, Listener listener, int bulkActions, TimeValue flushInterval) {
+    private BulkProcessor(ElasticsearchClient client, Listener listener, int bulkActions, TimeValue flushInterval) {
         this.bulkActions = bulkActions;
         this.bulkRequest = new BulkRequest();
         this.client = client;
@@ -90,11 +90,11 @@ public BulkProcessor add(DeleteRequest request) {
     /**
      * Adds either a delete or an index request.
      */
-    public BulkProcessor add(SingleBulkRequest request) {
+    private BulkProcessor add(SingleBulkRequest request) {
         return internalAdd(request);
     }
 
-    protected void ensureOpen() {
+    private void ensureOpen() {
         if (closed) {
             throw new IllegalStateException("bulk process already closed");
         }

src/main/java/fr/pilato/elasticsearch/crawler/fs/client/BulkResponse.java

@@ -44,7 +44,7 @@ public BulkItemResponse[] getItems() {
     }
 
     public Throwable buildFailureMessage() {
-        StringBuffer sbf = new StringBuffer();
+        StringBuilder sbf = new StringBuilder();
         int failures = 0;
         for (BulkItemResponse item : items) {
             if (item.failed) {
@@ -58,7 +58,7 @@ public Throwable buildFailureMessage() {
         if (logger.isTraceEnabled()) {
             sbf.append("\n");
         }
-        sbf.append(failures + " failures");
+        sbf.append(failures).append(" failures");
         return new RuntimeException(sbf.toString());
     }
 
@@ -120,7 +120,7 @@ public void setFailureMessage(String failureMessage) {
 
         @Override
         public String toString() {
-            final StringBuffer sb = new StringBuffer("BulkItemResponse{");
+            final StringBuilder sb = new StringBuilder("BulkItemResponse{");
             sb.append("failed=").append(failed);
             sb.append(", index='").append(index).append('\'');
             sb.append(", type='").append(type).append('\'');
@@ -134,7 +134,7 @@ public String toString() {
 
     @Override
     public String toString() {
-        final StringBuffer sb = new StringBuffer("BulkResponse{");
+        final StringBuilder sb = new StringBuilder("BulkResponse{");
         sb.append("items=").append(items == null ? "null" : Arrays.asList(items).toString());
         sb.append('}');
         return sb.toString();

src/main/java/fr/pilato/elasticsearch/crawler/fs/client/ElasticsearchClient.java

@@ -51,7 +51,7 @@ public class ElasticsearchClient {
 
     private static final Logger logger = LogManager.getLogger(ElasticsearchClient.class);
 
-    private List<Node> nodes;
+    private final List<Node> nodes;
 
     private ElasticsearchClient() {
         nodes = new ArrayList<>();

src/main/java/fr/pilato/elasticsearch/crawler/fs/client/SearchRequest.java

@@ -22,9 +22,9 @@
 import java.util.Arrays;
 
 public class SearchRequest {
-    private String query;
-    private String[] fields;
-    private Integer size;
+    private final String query;
+    private final String[] fields;
+    private final Integer size;
 
     public static Builder builder() {
         return new Builder();
@@ -50,7 +50,7 @@ public String[] getFields() {
 
     @Override
     public String   toString() {
-        final StringBuffer sb = new StringBuffer("SearchRequest{");
+        final StringBuilder sb = new StringBuilder("SearchRequest{");
         sb.append("query=").append(query);
         sb.append(", fields=").append(fields == null ? "null" : Arrays.asList(fields).toString());
         sb.append(", size=").append(size);

src/main/java/fr/pilato/elasticsearch/crawler/fs/client/SearchResponse.java

@@ -39,7 +39,7 @@ public Hits getHits() {
 
     @Override
     public String toString() {
-        final StringBuffer sb = new StringBuffer("SearchResponse{");
+        final StringBuilder sb = new StringBuilder("SearchResponse{");
         sb.append("hits=").append(hits);
         sb.append('}');
         return sb.toString();
@@ -63,7 +63,7 @@ public long getTotal() {
 
         @Override
         public String toString() {
-            final StringBuffer sb = new StringBuffer("Hits{");
+            final StringBuilder sb = new StringBuilder("Hits{");
             sb.append("hits=").append(hits);
             sb.append(", total=").append(total);
             sb.append('}');
@@ -88,7 +88,7 @@ public Map<String, Object> getFields() {
 
         @Override
         public String toString() {
-            final StringBuffer sb = new StringBuffer("Hit{");
+            final StringBuilder sb = new StringBuilder("Hit{");
             sb.append("source=").append(source);
             sb.append(", fields=").append(fields);
             sb.append('}');

src/main/java/fr/pilato/elasticsearch/crawler/fs/fileabstractor/FileAbstractModel.java

@@ -52,7 +52,7 @@ public class FileAbstractModel {
 
     @Override
     public String toString() {
-        final StringBuffer sb = new StringBuffer("FileAbstractModel{");
+        final StringBuilder sb = new StringBuilder("FileAbstractModel{");
         sb.append("name='").append(name).append('\'');
         sb.append(", file=").append(file);
         sb.append(", directory=").append(directory);

src/main/java/fr/pilato/elasticsearch/crawler/fs/fileabstractor/FileAbstractor.java

@@ -29,7 +29,7 @@
 public abstract class FileAbstractor<T> {
     protected static final Logger logger = LogManager.getLogger(FileAbstractor.class);
 
-    protected FsSettings fsSettings;
+    protected final FsSettings fsSettings;
 
     public abstract FileAbstractModel toFileAbstractModel(String path, T file);
 

src/main/java/fr/pilato/elasticsearch/crawler/fs/fileabstractor/FileAbstractorFile.java

@@ -26,8 +26,6 @@
 import java.io.FileInputStream;
 import java.io.InputStream;
 import java.time.Instant;
-import java.time.LocalDateTime;
-import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.Collection;
 
@@ -59,7 +57,7 @@ public Collection<FileAbstractModel> getFiles(String dir) {
         if (logger.isDebugEnabled()) logger.debug("Listing local files from {}", dir);
         File[] files = new File(dir).listFiles();
 
-        Collection<FileAbstractModel> result = new ArrayList<FileAbstractModel>(files.length);
+        Collection<FileAbstractModel> result = new ArrayList<>(files.length);
 
         // Iterate other files
         for (File file : files) {

src/main/java/fr/pilato/elasticsearch/crawler/fs/fileabstractor/FileAbstractorSSH.java

@@ -31,12 +31,13 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Vector;
+import java.util.stream.Collectors;
 
 public class FileAbstractorSSH extends FileAbstractor<ChannelSftp.LsEntry> {
 
     private ChannelSftp sftp;
 
-    public FileAbstractorSSH(FsSettings fsSettings) throws Exception {
+    public FileAbstractorSSH(FsSettings fsSettings) {
         super(fsSettings);
     }
 
@@ -65,15 +66,13 @@ public Collection<FileAbstractModel> getFiles(String dir) throws Exception {
         ls = sftp.ls(dir);
         if (ls == null) return null;
 
-        Collection<FileAbstractModel> result = new ArrayList<FileAbstractModel>(ls.size());
+        Collection<FileAbstractModel> result = new ArrayList<>(ls.size());
         // Iterate other files
-        for (ChannelSftp.LsEntry file : ls) {
-            // We ignore here all files like . and ..
-            if (!".".equals(file.getFilename()) &&
-                    !"..".equals(file.getFilename())) {
-                result.add(toFileAbstractModel(dir, file));
-            }
-        }
+        // We ignore here all files like . and ..
+        result.addAll(ls.stream().filter(file -> !".".equals(file.getFilename()) &&
+                !"..".equals(file.getFilename()))
+                .map(file -> toFileAbstractModel(dir, file))
+                .collect(Collectors.toList()));
 
         if (logger.isDebugEnabled()) logger.debug("{} local files found", result.size());
         return result;

src/main/java/fr/pilato/elasticsearch/crawler/fs/meta/settings/Elasticsearch.java

@@ -221,8 +221,7 @@ public boolean equals(Object o) {
         if (nodes != null ? !nodes.equals(that.nodes) : that.nodes != null) return false;
         if (index != null ? !index.equals(that.index) : that.index != null) return false;
         if (type != null ? !type.equals(that.type) : that.type != null) return false;
-        if (flushInterval != null ? !flushInterval.equals(that.flushInterval) : that.flushInterval != null) return false;
-        return true;
+        return !(flushInterval != null ? !flushInterval.equals(that.flushInterval) : that.flushInterval != null);
 
     }
 
@@ -231,7 +230,7 @@ public int hashCode() {
         int result = nodes != null ? nodes.hashCode() : 0;
         result = 31 * result + (index != null ? index.hashCode() : 0);
         result = 31 * result + (type != null ? type.hashCode() : 0);
-        result = 31 * result + (int) (bulkSize ^ (bulkSize >>> 32));
+        result = 31 * result + bulkSize;
         result = 31 * result + (flushInterval != null ? flushInterval.hashCode() : 0);
         return result;
     }

src/main/java/fr/pilato/elasticsearch/crawler/fs/meta/settings/PercentageDeserializer.java

@@ -20,7 +20,6 @@
 package fr.pilato.elasticsearch.crawler.fs.meta.settings;
 
 import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
 
@@ -35,7 +34,7 @@ public PercentageDeserializer() {
     }
 
     @Override
-    public Percentage deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+    public Percentage deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
         return Percentage.parse(p.getText(), null);
     }
 }

src/main/java/fr/pilato/elasticsearch/crawler/fs/meta/settings/TimeValueDeserializer.java

@@ -20,7 +20,6 @@
 package fr.pilato.elasticsearch.crawler.fs.meta.settings;
 
 import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
 
@@ -35,7 +34,7 @@ public TimeValueDeserializer() {
     }
 
     @Override
-    public TimeValue deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+    public TimeValue deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
         return TimeValue.parseTimeValue(p.getText(), null);
     }
 }

src/main/java/fr/pilato/elasticsearch/crawler/fs/util/FsCrawlerUtil.java

@@ -24,7 +24,6 @@
 import fr.pilato.elasticsearch.crawler.fs.meta.MetaParser;
 
 import java.io.File;
-import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -164,40 +163,40 @@ public static ObjectNode buildFsFileMapping() throws Exception {
         return buildFsFileMapping(true, false);
     }
 
-    private static void addAnalyzedString(ObjectNode node, String fieldName) throws IOException {
+    private static void addAnalyzedString(ObjectNode node, String fieldName) {
         node.putObject(fieldName)
                 .put("type", "string")
                 .put("store", "yes");
     }
 
-    private static void addNotAnalyzedString(ObjectNode node, String fieldName) throws IOException {
+    private static void addNotAnalyzedString(ObjectNode node, String fieldName) {
         node.putObject(fieldName)
                 .put("type", "string")
                 .put("store", "yes")
                 .put("index", "not_analyzed");
     }
 
-    private static void addNotIndexedString(ObjectNode node, String fieldName) throws IOException {
+    private static void addNotIndexedString(ObjectNode node, String fieldName) {
         node.putObject(fieldName)
                 .put("type", "string")
                 .put("store", "yes")
                 .put("index", "no");
     }
 
-    private static void addDate(ObjectNode node, String fieldName) throws IOException {
+    private static void addDate(ObjectNode node, String fieldName) {
         node.putObject(fieldName)
                 .put("type", "date")
                 .put("format", "dateOptionalTime")
                 .put("store", "yes");
     }
 
-    private static void addLong(ObjectNode node, String fieldName) throws IOException {
+    private static void addLong(ObjectNode node, String fieldName) {
         node.putObject(fieldName)
                 .put("type", "long")
                 .put("store", "yes");
     }
 
-    private static void addBinary(ObjectNode node, String fieldName) throws IOException {
+    private static void addBinary(ObjectNode node, String fieldName) {
         node.putObject(fieldName)
                 .put("type", "binary")
                 .put("store", "yes");