Release 3.1.6

Release notes are available at: https://docs.dremio.com/release-notes/31-release-notes.html Change-Id: Ie6c75a0c3ab74d0ddd0348eca0fa66503b359c58
shawnzhou66 · Mar 8, 2019 · 47d9901 · 47d9901
1 parent fcbdd77
commit 47d9901
Show file tree

Hide file tree

Showing 101 changed files with 532 additions and 195 deletions.
diff --git a/client/jdbc/pom.xml b/client/jdbc/pom.xml
@@ -21,7 +21,7 @@
  <parent>
  <groupId>com.dremio.client</groupId>
  <artifactId>dremio-client-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
  <artifactId>dremio-client-jdbc</artifactId>
  <name>Client - JDBC Driver</name>

diff --git a/client/pom.xml b/client/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-root</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <groupId>com.dremio.client</groupId>

diff --git a/common/pom.xml b/common/pom.xml
@@ -23,7 +23,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-root</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-common</artifactId>

diff --git a/contrib/hive-exec-shade/pom.xml b/contrib/hive-exec-shade/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio.contrib</groupId>
  <artifactId>dremio-contrib-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-hive-exec-shaded</artifactId>

diff --git a/contrib/pom.xml b/contrib/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-root</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <groupId>com.dremio.contrib</groupId>

diff --git a/contrib/slim-shade/pom.xml b/contrib/slim-shade/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio.contrib</groupId>
  <artifactId>dremio-contrib-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-slim-shade</artifactId>

diff --git a/contrib/tpch-sample-data/pom.xml b/contrib/tpch-sample-data/pom.xml
@@ -21,7 +21,7 @@
  <parent>
  <artifactId>dremio-contrib-parent</artifactId>
  <groupId>com.dremio.contrib</groupId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-tpch-sample-data</artifactId>

diff --git a/dac/backend/pom.xml b/dac/backend/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-dac-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-dac-backend</artifactId>

diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java
@@ -121,7 +121,7 @@ public SchemaMutability getSchemaMutability() {
 
  @Override
  public HomeFileSystemStoragePlugin newPlugin(SabotContext context, String name, Provider<StoragePluginId> pluginIdProvider) {
- return new HomeFileSystemStoragePlugin(this, context, name, null, pluginIdProvider);
+ return new HomeFileSystemStoragePlugin(this, context, name, pluginIdProvider);
  }
 
  public Path getBaseUploadsPath() {

diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java
@@ -66,8 +66,8 @@ public class HomeFileSystemStoragePlugin extends FileSystemPlugin<HomeFileConf>
  private final Path stagingDir;
  private final Path uploadsDir;
 
- public HomeFileSystemStoragePlugin(final HomeFileConf config, final SabotContext context, final String name, FileSystemWrapper fs, Provider<StoragePluginId> idProvider) {
- super(config, context, name, fs, idProvider);
+ public HomeFileSystemStoragePlugin(final HomeFileConf config, final SabotContext context, final String name, Provider<StoragePluginId> idProvider) {
+ super(config, context, name, idProvider);
  this.stagingDir = new Path(config.getPath(), STAGING + "." + context.getDremioConfig().getThisNode());
  this.uploadsDir = new Path(config.getPath(), UPLOADS);
  }
@@ -80,7 +80,7 @@ public HomeFileConf getConfig() {
  @Override
  public void start() throws IOException {
  super.start();
- FileSystem fs = getFs();
+ FileSystem fs = getSystemUserFS();
  fs.mkdirs(getConfig().getPath(), DEFAULT_PERMISSIONS);
  fs.mkdirs(stagingDir, DEFAULT_PERMISSIONS);
  fs.mkdirs(uploadsDir, DEFAULT_PERMISSIONS);
@@ -96,7 +96,7 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig
  PhysicalDataset physicalDataset = oldConfig == null ? null : oldConfig.getPhysicalDataset();
  FormatPluginConfig pluginConfig = null;
  try {
- final FileSystemWrapper fs = getFs();
+ final FileSystemWrapper fs = getSystemUserFS();
  final FileConfig fileConfig;
 
  if(physicalDataset != null && physicalDataset.getFormatSettings() != null){
@@ -113,7 +113,7 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig
  pluginConfig = PhysicalDatasetUtils.toFormatPlugin(fileConfig, Collections.<String>emptyList());
  final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(pluginConfig);
  return getDataset(datasetPath, oldConfig, formatPlugin, fs, fileConfig,
-  retrievalOptions.maxMetadataLeafColumns());
+ retrievalOptions.maxMetadataLeafColumns());
  } catch (NamespaceNotFoundException nfe){
  FormatPluginConfig formatPluginConfig = null;
  if(physicalDataset != null && physicalDataset.getFormatSettings() != null){
@@ -125,9 +125,9 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig
 
  @Override
  protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPluginConfig formatPluginConfig,
- DatasetRetrievalOptions retrievalOptions, String user) throws Exception {
+  DatasetRetrievalOptions retrievalOptions, String user) throws Exception {
  try{
- final FileSystemWrapper fs = getFs();
+ final FileSystemWrapper fs = getSystemUserFS();
  final DatasetConfig datasetConfig = getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath);
 
  if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) {
@@ -137,7 +137,7 @@ protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, D
  final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(formatPluginConfig);
 
  return getDataset(datasetPath, oldConfig, formatPlugin, fs,
-  datasetConfig.getPhysicalDataset().getFormatSettings(), retrievalOptions.maxMetadataLeafColumns());
+ datasetConfig.getPhysicalDataset().getFormatSettings(), retrievalOptions.maxMetadataLeafColumns());
  } catch (NamespaceNotFoundException nfe){
  if(formatPluginConfig == null) {
  // a home file can only be read from the namespace or using a format options. Without either, it is invalid, return nothing.
@@ -152,23 +152,23 @@ protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, D
  public boolean containerExists(NamespaceKey key) {
  List<String> folderPath = key.getPathComponents();
  try {
- return getFS(SystemUser.SYSTEM_USERNAME).exists(PathUtils.toFSPath(
- ImmutableList.<String>builder()
- .addAll(folderPath.subList(1, folderPath.size()))
- .build()));
+  return getSystemUserFS().exists(PathUtils.toFSPath(
+  ImmutableList.<String>builder()
+  .addAll(folderPath.subList(1, folderPath.size()))
+  .build()));
  }catch(IOException e) {
  logger.info("IOException while trying to retrieve home files.", e);
  return false;
  }
  }
 
  private SourceTableDefinition getDataset(
-  NamespaceKey datasetPath,
-  DatasetConfig oldConfig,
-  FormatPlugin formatPlugin,
-  FileSystemWrapper fs,
-  FileConfig fileConfig,
-  int maxLeafColumns
+ NamespaceKey datasetPath,
+ DatasetConfig oldConfig,
+ FormatPlugin formatPlugin,
+ FileSystemWrapper fs,
+ FileConfig fileConfig,
+ int maxLeafColumns
  ) throws IOException {
 
  final List<FileSystemCachedEntity> cachedEntities = Lists.newArrayList();

diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java
@@ -55,7 +55,7 @@ public HomeFileTool(SabotContext context, CatalogService catalog) throws Executi
  StoragePlugin plugin = catalog.getSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME);
  Preconditions.checkNotNull(plugin, "Plugin [%s] not found.", HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME);
  HomeFileSystemStoragePlugin homePlugin = (HomeFileSystemStoragePlugin) plugin;
- this.fs = homePlugin.getProcessFs();
+ this.fs = homePlugin.getSystemUserFS();
  this.config = homePlugin.getConfig();
  this.hostname = context.getDremioConfig().getThisNode();
  }

diff --git a/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java b/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java
@@ -168,7 +168,7 @@ public FileFormat getOrDetectFormat(NamespacePath folderPath, DatasetType expect
 
  private FileFormat detectFileFormat(NamespaceKey key) {
  final FileSystemPlugin plugin = getPlugin(key);
- final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName());
+ final FileSystemWrapper fs = plugin.createFS(securityContext.getUserPrincipal().getName());
  final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents()));
 
  // for now, existing rudimentary behavior that uses extension detection.
@@ -240,7 +240,7 @@ private static FileFormat asFormat(NamespaceKey key, Path path, boolean isFolder
  public JobDataFragment previewData(FileFormat format, NamespacePath namespacePath, boolean useFormatLocation) {
  final NamespaceKey key = namespacePath.toNamespaceKey();
  final FileSystemPlugin plugin = getPlugin(key);
- final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName());
+ final FileSystemWrapper fs = plugin.createFS(securityContext.getUserPrincipal().getName());
  final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents()));
 
  // for now, existing rudimentary behavior that uses extension detection.
@@ -342,7 +342,7 @@ private JobDataFragment getData(FileFormat format, FileSystemPlugin plugin, File
  if (first) {
  first = false;
  container.buildSchema();
- if (container.getSchema().getFieldCount() > maxLeafColumns) {
+ if (container.getSchema().getTotalFieldCount() > maxLeafColumns) {
  throw UserException.validationError()
  .message("Using datasets with more than %d columns is currently disabled.", maxLeafColumns)
  .build(logger);

diff --git a/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java b/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java
@@ -110,7 +110,8 @@ public DatasetVersionMutator(
 
  private DatasetDownloadManager downloadManager() {
  final FileSystemPlugin downloadPlugin = catalogService.getSource(DATASET_DOWNLOAD_STORAGE_PLUGIN);
- return new DatasetDownloadManager(jobsService, namespaceService, downloadPlugin.getConfig().getPath(), downloadPlugin.getFs());
+ return new DatasetDownloadManager(jobsService, namespaceService, downloadPlugin.getConfig().getPath(),
+ downloadPlugin.getSystemUserFS());
  }
  private void validate(DatasetPath path, VirtualDatasetUI ds) {
  if (ds.getSqlFieldsList() == null || ds.getSqlFieldsList().isEmpty()) {

diff --git a/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java b/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java
@@ -73,7 +73,7 @@ public SchemaMutability getSchemaMutability() {
 
  @Override
  public FileSystemPlugin<APrivateSource> newPlugin(SabotContext context, String name, Provider<StoragePluginId> pluginIdProvider) {
- return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider);
+ return new FileSystemPlugin<>(this, context, name, pluginIdProvider);
  }
 
 

diff --git a/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java b/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java
@@ -32,8 +32,11 @@
 import org.apache.hadoop.fs.Path;
 import org.junit.Before;
 import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 
+import com.dremio.common.exceptions.UserException;
 import com.dremio.common.util.FileUtils;
 import com.dremio.common.utils.PathUtils;
 import com.dremio.dac.explore.model.FileFormatUI;
@@ -73,6 +76,8 @@
  * Tests to create, update and execute queries on physical datasets..
  */
 public class TestPhysicalDatasets extends BaseTestServer {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
 
  @Before
  public void setup() throws Exception {
@@ -294,6 +299,30 @@ public void testSchemaChangeExit() throws Exception {
  assertEquals(1, data.getColumns().size());
  }
 
+ @Test
+ public void testLargeJsonFile() throws Exception {
+ final JobsService jobsService = l(JobsService.class);
+ JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder()
+ .setSqlQuery(createQuery("/datasets/wide_table.json"))
+ .setQueryType(QueryType.UI_RUN)
+ .build(), NoOpJobStatusListener.INSTANCE));
+
+ thrown.expect(UserException.class);
+ job.getData().truncate(500);
+ }
+
+ @Test
+ public void testLargeNestedJsonFile() throws Exception {
+ final JobsService jobsService = l(JobsService.class);
+ JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder()
+ .setSqlQuery(createQuery("/datasets/wide_nested_table.json"))
+ .setQueryType(QueryType.UI_RUN)
+ .build(), NoOpJobStatusListener.INSTANCE));
+
+ thrown.expect(UserException.class);
+ job.getData().truncate(500);
+ }
+
  @Test
  public void testParquetFile() throws Exception {
  final JobsService jobsService = l(JobsService.class);

diff --git a/dac/backend/src/test/resources/datasets/wide_nested_table.json b/dac/backend/src/test/resources/datasets/wide_nested_table.json
diff --git a/dac/backend/src/test/resources/datasets/wide_table.json b/dac/backend/src/test/resources/datasets/wide_table.json
diff --git a/dac/common/pom.xml b/dac/common/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-dac-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-dac-common</artifactId>

diff --git a/dac/daemon/pom.xml b/dac/daemon/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-dac-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-dac-daemon</artifactId>

diff --git a/dac/pom.xml b/dac/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-root</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-dac-parent</artifactId>

diff --git a/dac/ui/pom.xml b/dac/ui/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-dac-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-dac-ui</artifactId>

diff --git a/distribution/jdbc-driver/pom.xml b/distribution/jdbc-driver/pom.xml
@@ -21,7 +21,7 @@
  <parent>
  <groupId>com.dremio.distribution</groupId>
  <artifactId>dremio-distribution-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-jdbc-driver</artifactId>

diff --git a/distribution/pom.xml b/distribution/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio</groupId>
  <artifactId>dremio-root</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <groupId>com.dremio.distribution</groupId>

diff --git a/distribution/resources/pom.xml b/distribution/resources/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio.distribution</groupId>
  <artifactId>dremio-distribution-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-distribution-resources</artifactId>

diff --git a/distribution/resources/src/main/resources/bin/dremio b/distribution/resources/src/main/resources/bin/dremio
@@ -55,6 +55,15 @@ shift
 
 command=dremio
 
+print_warn() {
+ echo "#######################################################################"
+ echo "WARNING:"
+ echo ""
+ echo -e "$1"
+ echo ""
+ echo "#######################################################################"
+}
+
 wait_for_process_end() {
  pidKilled=$1
  commandName=$2
@@ -188,13 +197,22 @@ DREMIO_NICENESS="${DREMIO_NICENESS:-0}"
 thiscmd=$0
 args=$@
 
+ULIMIT=65536
+if [ "$(ulimit -n)" -lt "$ULIMIT" ]; then
+ print_warn "\
+Please increase the maximum number of file descriptors to at least $ULIMIT.\n\n\
+Dremio utilizes a large number of file descriptors.\n\
+The maximum number of file descriptors, currently set at $(ulimit -n), is not sufficient."
+fi
+
 case $startStopStatus in
 
 (start)
  check_before_start
- echo starting $command, logging to $logout
+ echo Starting $command, logging to $logout
  nohup $thiscmd internal_start $args < /dev/null >> ${logout} 2>&1 &
  sleep 1;
+ echo Check the log file to make sure $command started successfully and did not run into any issues.
  ;;
 
 (start-fg)

diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
@@ -22,7 +22,7 @@
  <parent>
  <groupId>com.dremio.distribution</groupId>
  <artifactId>dremio-distribution-parent</artifactId>
- <version>3.1.4-201902210617130445-0d64760</version>
+ <version>3.1.6-201903070042400578-fdcd3a8</version>
  </parent>
 
  <artifactId>dremio-distribution</artifactId>