Make using implicit system Charset an error (apache#4326)

* Make using implicit system charset an error * Use StringUtils.toUtf8() and fromUtf8() instead of String.getBytes() and new String() * Use English locale in StringUtils.safeFormat() * Restore comment
caobaohe · Jun 6, 2017 · 31d33b3 · 31d33b3
1 parent ada498c
commit 31d33b3
Show file tree

Hide file tree

Showing 89 changed files with 382 additions and 324 deletions.
diff --git a/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java b/api/src/test/java/io/druid/data/input/impl/InputRowParserSerdeTest.java
@@ -27,6 +27,7 @@
 import io.druid.TestObjectMapper;
 import io.druid.data.input.ByteBufferInputRowParser;
 import io.druid.data.input.InputRow;
+import io.druid.java.util.common.StringUtils;
 import org.joda.time.DateTime;
 import org.junit.Assert;
 import org.junit.Test;
@@ -57,9 +58,7 @@ public void testStringInputRowParserSerde() throws Exception
         ByteBufferInputRowParser.class
     );
     final InputRow parsed = parser2.parse(
-        ByteBuffer.wrap(
-            "{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"2000\"}".getBytes(Charsets.UTF_8)
-        )
+        ByteBuffer.wrap(StringUtils.toUtf8("{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"2000\"}"))
     );
     Assert.assertEquals(ImmutableList.of("foo", "bar"), parsed.getDimensions());
     Assert.assertEquals(ImmutableList.of("x"), parsed.getDimension("foo"));

diff --git a/api/src/test/java/io/druid/data/input/impl/PrefetchableTextFilesFirehoseFactoryTest.java b/api/src/test/java/io/druid/data/input/impl/PrefetchableTextFilesFirehoseFactoryTest.java
@@ -35,12 +35,12 @@
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -89,9 +89,7 @@ public static void setup() throws IOException
 
     for (int i = 0; i < 10; i++) {
       // Each file is 1390 bytes
-      try (final Writer writer = new BufferedWriter(
-          new FileWriter(new File(testDir, "test_" + i))
-      )) {
+      try (final Writer writer = Files.newBufferedWriter(new File(testDir, "test_" + i).toPath(), StandardCharsets.UTF_8)) {
         for (int j = 0; j < 100; j++) {
           final String a = (20171220 + i) + "," + i + "," + j + "\n";
           writer.write(a);

diff --git a/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmarkFileGenerator.java b/benchmarks/src/main/java/io/druid/benchmark/FloatCompressionBenchmarkFileGenerator.java
@@ -30,20 +30,17 @@
 import io.druid.segment.data.TmpFileIOPeon;
 
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.OutputStream;
-import java.io.OutputStreamWriter;
 import java.io.Writer;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
 import java.util.HashMap;
 import java.util.List;
@@ -135,7 +132,7 @@ public static void main(String[] args) throws IOException, URISyntaxException
     for (Map.Entry<String, BenchmarkColumnValueGenerator> entry : generators.entrySet()) {
       final File dataFile = new File(dir, entry.getKey());
       dataFile.delete();
-      try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile)))) {
+      try (Writer writer = Files.newBufferedWriter(dataFile.toPath(), StandardCharsets.UTF_8)) {
         for (int i = 0; i < ROW_NUM; i++) {
           writer.write((Float) entry.getValue().generateRowValue() + "\n");
         }
@@ -158,7 +155,7 @@ public static void main(String[] args) throws IOException, URISyntaxException
             ByteOrder.nativeOrder(),
             compression
         );
-        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(dataFile)));
+        BufferedReader br = Files.newBufferedReader(dataFile.toPath(), StandardCharsets.UTF_8);
 
         try (FileChannel output = FileChannel.open(
             compFile.toPath(),

diff --git a/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmarkFileGenerator.java b/benchmarks/src/main/java/io/druid/benchmark/LongCompressionBenchmarkFileGenerator.java
@@ -30,20 +30,17 @@
 import io.druid.segment.data.TmpFileIOPeon;
 
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.OutputStream;
-import java.io.OutputStreamWriter;
 import java.io.Writer;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.nio.file.StandardOpenOption;
 import java.util.HashMap;
 import java.util.List;
@@ -126,7 +123,7 @@ public static void main(String[] args) throws IOException, URISyntaxException
     for (Map.Entry<String, BenchmarkColumnValueGenerator> entry : generators.entrySet()) {
       final File dataFile = new File(dir, entry.getKey());
       dataFile.delete();
-      try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(dataFile)))) {
+      try (Writer writer = Files.newBufferedWriter(dataFile.toPath(), StandardCharsets.UTF_8)) {
         for (int i = 0; i < ROW_NUM; i++) {
           writer.write((long) entry.getValue().generateRowValue() + "\n");
         }
@@ -151,7 +148,7 @@ public static void main(String[] args) throws IOException, URISyntaxException
               encoding,
               compression
           );
-          BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(dataFile)));
+          BufferedReader br = Files.newBufferedReader(dataFile.toPath(), StandardCharsets.UTF_8);
 
           try (FileChannel output = FileChannel.open(
               compFile.toPath(),

diff --git a/benchmarks/src/main/java/io/druid/benchmark/VSizeSerdeBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/VSizeSerdeBenchmark.java
@@ -34,13 +34,12 @@
 import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
 
-import java.io.BufferedWriter;
 import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
 import java.io.Writer;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.TimeUnit;
 
 @State(Scope.Benchmark)
@@ -77,7 +76,7 @@ public void setup() throws IOException, URISyntaxException
     // to construct a heapByteBuffer since they have different performance
     File base = new File(this.getClass().getClassLoader().getResource("").toURI());
     dummy = new File(base, "dummy");
-    try (Writer writer = new BufferedWriter(new FileWriter(dummy))) {
+    try (Writer writer = java.nio.file.Files.newBufferedWriter(dummy.toPath(), StandardCharsets.UTF_8)) {
       String EMPTY_STRING = "        ";
       for (int i = 0; i < values + 10; i++) {
         writer.write(EMPTY_STRING);

diff --git a/common/src/main/java/io/druid/common/utils/SerializerUtils.java b/common/src/main/java/io/druid/common/utils/SerializerUtils.java
@@ -25,20 +25,19 @@
 import com.google.common.primitives.Ints;
 import com.google.common.primitives.Longs;
 import io.druid.collections.IntList;
+import io.druid.java.util.common.StringUtils;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.channels.WritableByteChannel;
-import java.nio.charset.Charset;
 import java.util.Arrays;
 import java.util.List;
 
 public class SerializerUtils
 {
-  private static final Charset UTF8 = Charset.forName("UTF-8");
 
   /**
    * Writes the given long value into the given OutputStream in big-endian byte order, using the helperBuffer. Faster
@@ -106,7 +105,7 @@ public static void writeNativeOrderedIntToOutputStream(OutputStream out, int val
 
   public <T extends OutputStream> void writeString(T out, String name) throws IOException
   {
-    byte[] nameBytes = name.getBytes(UTF8);
+    byte[] nameBytes = StringUtils.toUtf8(name);
     writeInt(out, nameBytes.length);
     out.write(nameBytes);
   }
@@ -120,7 +119,7 @@ public void writeString(OutputSupplier<? extends OutputStream> supplier, String
 
   public void writeString(WritableByteChannel out, String name) throws IOException
   {
-    byte[] nameBytes = name.getBytes(UTF8);
+    byte[] nameBytes = StringUtils.toUtf8(name);
     writeInt(out, nameBytes.length);
     out.write(ByteBuffer.wrap(nameBytes));
   }
@@ -322,6 +321,6 @@ public float[] readFloats(InputStream in) throws IOException
 
   public int getSerializedStringByteSize(String str)
   {
-    return Ints.BYTES + str.getBytes(UTF8).length;
+    return Ints.BYTES + StringUtils.toUtf8(str).length;
   }
 }
diff --git a/common/src/main/java/io/druid/common/utils/ServletResourceUtils.java b/common/src/main/java/io/druid/common/utils/ServletResourceUtils.java
@@ -20,6 +20,7 @@
 package io.druid.common.utils;
 
 import com.google.common.collect.ImmutableMap;
+import io.druid.java.util.common.StringUtils;
 
 import javax.annotation.Nullable;
 import java.util.Map;

diff --git a/common/src/main/java/io/druid/common/utils/StringUtils.java b/common/src/main/java/io/druid/common/utils/StringUtils.java
diff --git a/common/src/main/java/io/druid/math/expr/Parser.java b/common/src/main/java/io/druid/math/expr/Parser.java
@@ -23,10 +23,9 @@
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-import com.google.common.collect.Lists;
-
 import io.druid.java.util.common.IAE;
 import io.druid.java.util.common.logger.Logger;
 import io.druid.math.expr.antlr.ExprLexer;

diff --git a/common/src/test/java/io/druid/common/utils/SerializerUtilsTest.java b/common/src/test/java/io/druid/common/utils/SerializerUtilsTest.java
@@ -20,6 +20,7 @@
 package io.druid.common.utils;
 
 import io.druid.collections.IntList;
+import io.druid.java.util.common.StringUtils;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -32,7 +33,6 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
-import java.nio.charset.Charset;
 
 public class SerializerUtilsTest
 {
@@ -42,7 +42,6 @@ public class SerializerUtilsTest
   private  final int [] ints = {1,2,3};
   private  final float [] floats = {1.1f,2,3};
   private  final long [] longs = {3,2,1};
-  private  final Charset UTF8 = Charset.forName("UTF-8");
   private  byte [] stringsByte;
   private  byte [] intsByte;
   private  byte [] floatsByte;
@@ -56,9 +55,9 @@ public void setUpByteArrays() throws IOException
     DataOutputStream out = new DataOutputStream(bos);
     out.writeInt(strings.length);
     for (int i = 0;i < strings.length;i++) {
-      byte [] stringBytes = strings[i].getBytes(UTF8);
+      byte [] stringBytes = StringUtils.toUtf8(strings[i]);
       out.writeInt(stringBytes.length);
-      out.write(strings[i].getBytes());
+      out.write(StringUtils.toUtf8(strings[i]));
     }
     out.close();
     stringsByte = bos.toByteArray();

diff --git a/common/src/test/java/io/druid/common/utils/StringUtilsTest.java b/common/src/test/java/io/druid/common/utils/StringUtilsTest.java
@@ -19,6 +19,7 @@
 
 package io.druid.common.utils;
 
+import io.druid.java.util.common.StringUtils;
 import org.junit.Assert;
 import org.junit.Test;
 

diff --git a/...ain/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java b/...ain/java/io/druid/emitter/ambari/metrics/WhiteListBasedDruidToTimelineEventConverter.java
@@ -39,8 +39,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
 import java.util.SortedMap;
@@ -199,11 +200,10 @@ private ImmutableSortedMap<String, ImmutableList<String>> readMap(final String m
       if (Strings.isNullOrEmpty(mapPath)) {
         actualPath = this.getClass().getClassLoader().getResource("defaultWhiteListMap.json").getFile();
         LOGGER.info("using default whiteList map located at [%s]", actualPath);
-        fileContent = CharStreams.toString(new InputStreamReader(this.getClass()
-                                                                     .getClassLoader()
-                                                                     .getResourceAsStream("defaultWhiteListMap.json")));
+        InputStream byteContent = this.getClass().getClassLoader().getResourceAsStream("defaultWhiteListMap.json");
+        fileContent = CharStreams.toString(new InputStreamReader(byteContent, StandardCharsets.UTF_8));
       } else {
-        fileContent = Files.asCharSource(new File(mapPath), Charset.forName("UTF-8")).read();
+        fileContent = Files.asCharSource(new File(mapPath), StandardCharsets.UTF_8).read();
       }
       return mapper.reader(new TypeReference<ImmutableSortedMap<String, ImmutableList<String>>>()
       {

diff --git a/...ions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java b/...ions-contrib/azure-extensions/src/test/java/io/druid/storage/azure/AzureTaskLogsTest.java
@@ -23,6 +23,7 @@
 import com.google.common.base.Optional;
 import com.google.common.io.ByteSource;
 import com.google.common.io.Files;
+import io.druid.java.util.common.StringUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.easymock.EasyMockSupport;
@@ -126,7 +127,7 @@ public void testStreamTaskLogWithNegative() throws Exception {
     expect(azureStorage.getBlobExists(container, blobPath)).andReturn(true);
     expect(azureStorage.getBlobLength(container, blobPath)).andReturn((long) testLog.length());
     expect(azureStorage.getBlobInputStream(container, blobPath)).andReturn(
-        new ByteArrayInputStream(testLog.getBytes(Charsets.UTF_8)));
+        new ByteArrayInputStream(StringUtils.toUtf8(testLog)));
 
 
     replayAll();