File.deleteOnExit() (apache#3923)

* Less use of File.deleteOnExit() * removed deleteOnExit from most of the tests/benchmarks/iopeon * Made IOpeon closable * Formatting. * Revert DeterminePartitionsJobTest, remove cleanup method from IOPeon
Gamehu · Feb 13, 2017 · 8854ce0 · 8854ce0
1 parent 9dfcf07
commit 8854ce0
Show file tree

Hide file tree

Showing 34 changed files with 478 additions and 340 deletions.
diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilterPartitionBenchmark.java
@@ -26,7 +26,6 @@
 import com.google.common.collect.Lists;
 import com.google.common.hash.Hashing;
 import com.google.common.io.Files;
-
 import io.druid.benchmark.datagen.BenchmarkDataGenerator;
 import io.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import io.druid.benchmark.datagen.BenchmarkSchemas;
@@ -76,6 +75,7 @@
 import io.druid.segment.incremental.IncrementalIndexSchema;
 import io.druid.segment.incremental.OnheapIncrementalIndex;
 import io.druid.segment.serde.ComplexMetrics;
+import org.apache.commons.io.FileUtils;
 import org.joda.time.Interval;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -87,6 +87,7 @@
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -118,6 +119,7 @@ public class FilterPartitionBenchmark
   private IncrementalIndex incIndex;
   private QueryableIndex qIndex;
   private File indexFile;
+  private File tmpDir;
 
   private Filter timeFilterNone;
   private Filter timeFilterHalf;
@@ -172,13 +174,12 @@ public void setup() throws IOException
       incIndex.add(row);
     }
 
-    File tmpFile = Files.createTempDir();
-    log.info("Using temp dir: " + tmpFile.getAbsolutePath());
-    tmpFile.deleteOnExit();
+    tmpDir = Files.createTempDir();
+    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
     indexFile = INDEX_MERGER_V9.persist(
         incIndex,
-        tmpFile,
+        tmpDir,
         new IndexSpec()
     );
     qIndex = INDEX_IO.loadIndex(indexFile);
@@ -219,6 +220,12 @@ public void setup() throws IOException
     ));
   }
 
+  @TearDown
+  public void tearDown() throws IOException
+  {
+    FileUtils.deleteDirectory(tmpDir);
+  }
+
   private IncrementalIndex makeIncIndex()
   {
     return new OnheapIncrementalIndex(

diff --git a/benchmarks/src/main/java/io/druid/benchmark/FilteredAggregatorBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/FilteredAggregatorBenchmark.java
@@ -24,7 +24,6 @@
 import com.google.common.collect.Maps;
 import com.google.common.hash.Hashing;
 import com.google.common.io.Files;
-
 import io.druid.benchmark.datagen.BenchmarkDataGenerator;
 import io.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import io.druid.benchmark.datagen.BenchmarkSchemas;
@@ -77,6 +76,7 @@
 import io.druid.segment.incremental.IncrementalIndexSchema;
 import io.druid.segment.incremental.OnheapIncrementalIndex;
 import io.druid.segment.serde.ComplexMetrics;
+import org.apache.commons.io.FileUtils;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -87,6 +87,7 @@
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -124,6 +125,7 @@ public class FilteredAggregatorBenchmark
   private QueryRunnerFactory factory;
   private BenchmarkSchemaInfo schemaInfo;
   private TimeseriesQuery query;
+  private File tmpDir;
 
   private static String JS_FN = "function(str) { return 'super-' + str; }";
   private static ExtractionFn JS_EXTRACTION_FN = new JavaScriptExtractionFn(JS_FN, false, JavaScriptConfig.getEnabledInstance());
@@ -187,13 +189,12 @@ public void setup() throws IOException
       inputRows.add(row);
     }
 
-    File tmpFile = Files.createTempDir();
-    log.info("Using temp dir: " + tmpFile.getAbsolutePath());
-    tmpFile.deleteOnExit();
+    tmpDir = Files.createTempDir();
+    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
     indexFile = INDEX_MERGER_V9.persist(
         incIndex,
-        tmpFile,
+        tmpDir,
         new IndexSpec()
     );
     qIndex = INDEX_IO.loadIndex(indexFile);
@@ -220,6 +221,12 @@ public void setup() throws IOException
                   .build();
   }
 
+  @TearDown
+  public void tearDown() throws IOException
+  {
+    FileUtils.deleteDirectory(tmpDir);
+  }
+
   private IncrementalIndex makeIncIndex(AggregatorFactory[] metrics)
   {
     return new OnheapIncrementalIndex(

diff --git a/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmarkFileGenerator.java b/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmarkFileGenerator.java
@@ -184,7 +184,7 @@ public OutputStream openStream() throws IOException
           output.write(ByteBuffer.wrap(baos.toByteArray()));
         }
         finally {
-          iopeon.cleanup();
+          iopeon.close();
           br.close();
         }
         System.out.print(compFile.length() / 1024 + "\n");

diff --git a/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmarkFileGenerator.java b/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmarkFileGenerator.java
@@ -177,7 +177,7 @@ public OutputStream openStream() throws IOException
             output.write(ByteBuffer.wrap(baos.toByteArray()));
           }
           finally {
-            iopeon.cleanup();
+            iopeon.close();
             br.close();
           }
           System.out.print(compFile.length() / 1024 + "\n");

diff --git a/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexMergeBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexMergeBenchmark.java
@@ -22,7 +22,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.hash.Hashing;
 import com.google.common.io.Files;
-
 import io.druid.benchmark.datagen.BenchmarkDataGenerator;
 import io.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import io.druid.benchmark.datagen.BenchmarkSchemas;
@@ -42,6 +41,7 @@
 import io.druid.segment.incremental.IncrementalIndexSchema;
 import io.druid.segment.incremental.OnheapIncrementalIndex;
 import io.druid.segment.serde.ComplexMetrics;
+import org.apache.commons.io.FileUtils;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -52,6 +52,7 @@
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -88,6 +89,7 @@ public class IndexMergeBenchmark
 
   private List<QueryableIndex> indexesToMerge;
   private BenchmarkSchemaInfo schemaInfo;
+  private File tmpDir;
 
   static {
     JSON_MAPPER = new DefaultObjectMapper();
@@ -137,13 +139,12 @@ public void setup() throws IOException
         incIndex.add(row);
       }
 
-      File tmpFile = Files.createTempDir();
-      log.info("Using temp dir: " + tmpFile.getAbsolutePath());
-      tmpFile.deleteOnExit();
+      tmpDir = Files.createTempDir();
+      log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
       File indexFile = INDEX_MERGER_V9.persist(
           incIndex,
-          tmpFile,
+          tmpDir,
           new IndexSpec()
       );
 
@@ -152,6 +153,12 @@ public void setup() throws IOException
     }
   }
 
+  @TearDown
+  public void tearDown() throws IOException
+  {
+    FileUtils.deleteDirectory(tmpDir);
+  }
+
   private IncrementalIndex makeIncIndex()
   {
     return new OnheapIncrementalIndex(
@@ -176,14 +183,23 @@ public void merge(Blackhole blackhole) throws Exception
     File tmpFile = File.createTempFile("IndexMergeBenchmark-MERGEDFILE-" + System.currentTimeMillis(), ".TEMPFILE");
     tmpFile.delete();
     tmpFile.mkdirs();
-    log.info(tmpFile.getAbsolutePath() + " isFile: " + tmpFile.isFile() + " isDir:" + tmpFile.isDirectory());
-    tmpFile.deleteOnExit();
+    try {
+      log.info(tmpFile.getAbsolutePath() + " isFile: " + tmpFile.isFile() + " isDir:" + tmpFile.isDirectory());
 
-    File mergedFile = INDEX_MERGER.mergeQueryableIndex(indexesToMerge, rollup, schemaInfo.getAggsArray(), tmpFile, new IndexSpec());
+      File mergedFile = INDEX_MERGER.mergeQueryableIndex(
+          indexesToMerge,
+          rollup,
+          schemaInfo.getAggsArray(),
+          tmpFile,
+          new IndexSpec()
+      );
 
-    blackhole.consume(mergedFile);
+      blackhole.consume(mergedFile);
+    }
+    finally {
+      tmpFile.delete();
+    }
 
-    tmpFile.delete();
   }
 
   @Benchmark
@@ -194,13 +210,23 @@ public void mergeV9(Blackhole blackhole) throws Exception
     File tmpFile = File.createTempFile("IndexMergeBenchmark-MERGEDFILE-V9-" + System.currentTimeMillis(), ".TEMPFILE");
     tmpFile.delete();
     tmpFile.mkdirs();
-    log.info(tmpFile.getAbsolutePath() + " isFile: " + tmpFile.isFile() + " isDir:" + tmpFile.isDirectory());
-    tmpFile.deleteOnExit();
+    try {
+      log.info(tmpFile.getAbsolutePath() + " isFile: " + tmpFile.isFile() + " isDir:" + tmpFile.isDirectory());
 
-    File mergedFile = INDEX_MERGER_V9.mergeQueryableIndex(indexesToMerge, rollup, schemaInfo.getAggsArray(), tmpFile, new IndexSpec());
+      File mergedFile = INDEX_MERGER_V9.mergeQueryableIndex(
+          indexesToMerge,
+          rollup,
+          schemaInfo.getAggsArray(),
+          tmpFile,
+          new IndexSpec()
+      );
 
-    blackhole.consume(mergedFile);
+      blackhole.consume(mergedFile);
+    }
+    finally {
+      tmpFile.delete();
+
+    }
 
-    tmpFile.delete();
   }
 }
diff --git a/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexPersistBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/indexing/IndexPersistBenchmark.java
@@ -22,7 +22,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.hash.Hashing;
 import com.google.common.io.Files;
-
 import io.druid.benchmark.datagen.BenchmarkDataGenerator;
 import io.druid.benchmark.datagen.BenchmarkSchemaInfo;
 import io.druid.benchmark.datagen.BenchmarkSchemas;
@@ -41,6 +40,7 @@
 import io.druid.segment.incremental.IncrementalIndexSchema;
 import io.druid.segment.incremental.OnheapIncrementalIndex;
 import io.druid.segment.serde.ComplexMetrics;
+import org.apache.commons.io.FileUtils;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -174,38 +174,42 @@ private IncrementalIndex makeIncIndex()
   @OutputTimeUnit(TimeUnit.MICROSECONDS)
   public void persist(Blackhole blackhole) throws Exception
   {
-    File tmpFile = Files.createTempDir();
-    log.info("Using temp dir: " + tmpFile.getAbsolutePath());
-    tmpFile.deleteOnExit();
-
-    File indexFile = INDEX_MERGER.persist(
-        incIndex,
-        tmpFile,
-        new IndexSpec()
-    );
-
-    blackhole.consume(indexFile);
+    File tmpDir = Files.createTempDir();
+    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
+    try {
+      File indexFile = INDEX_MERGER.persist(
+          incIndex,
+          tmpDir,
+          new IndexSpec()
+      );
+
+      blackhole.consume(indexFile);
+    }
+    finally {
+      FileUtils.deleteDirectory(tmpDir);
+    }
 
-    tmpFile.delete();
   }
 
   @Benchmark
   @BenchmarkMode(Mode.AverageTime)
   @OutputTimeUnit(TimeUnit.MICROSECONDS)
   public void persistV9(Blackhole blackhole) throws Exception
   {
-    File tmpFile = Files.createTempDir();
-    log.info("Using temp dir: " + tmpFile.getAbsolutePath());
-    tmpFile.deleteOnExit();;
-
-    File indexFile = INDEX_MERGER_V9.persist(
-        incIndex,
-        tmpFile,
-        new IndexSpec()
-    );
+    File tmpDir = Files.createTempDir();
+    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
+    try {
+      File indexFile = INDEX_MERGER_V9.persist(
+          incIndex,
+          tmpDir,
+          new IndexSpec()
+      );
 
-    blackhole.consume(indexFile);
+      blackhole.consume(indexFile);
 
-    tmpFile.delete();
+    }
+    finally {
+    FileUtils.deleteDirectory(tmpDir);
+    }
   }
 }
diff --git a/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/query/SearchBenchmark.java
@@ -79,6 +79,7 @@
 import io.druid.segment.incremental.IncrementalIndexSchema;
 import io.druid.segment.incremental.OnheapIncrementalIndex;
 import io.druid.segment.serde.ComplexMetrics;
+import org.apache.commons.io.FileUtils;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
@@ -89,6 +90,7 @@
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 import org.openjdk.jmh.infra.Blackhole;
 
@@ -132,6 +134,7 @@ public class SearchBenchmark
   private BenchmarkSchemaInfo schemaInfo;
   private Druids.SearchQueryBuilder queryBuilder;
   private SearchQuery query;
+  private File tmpDir;
 
   private ExecutorService executorService;
 
@@ -351,15 +354,14 @@ public void setup() throws IOException
       incIndexes.add(incIndex);
     }
 
-    File tmpFile = Files.createTempDir();
-    log.info("Using temp dir: " + tmpFile.getAbsolutePath());
-    tmpFile.deleteOnExit();
+    tmpDir = Files.createTempDir();
+    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
 
     qIndexes = new ArrayList<>();
     for (int i = 0; i < numSegments; i++) {
       File indexFile = INDEX_MERGER_V9.persist(
           incIndexes.get(i),
-          tmpFile,
+          tmpDir,
           new IndexSpec()
       );
 
@@ -378,6 +380,12 @@ public void setup() throws IOException
     );
   }
 
+  @TearDown
+  public void tearDown() throws IOException
+  {
+    FileUtils.deleteDirectory(tmpDir);
+  }
+
   private IncrementalIndex makeIncIndex()
   {
     return new OnheapIncrementalIndex(