diff --git a/client/jdbc/pom.xml b/client/jdbc/pom.xml
index 33f5fd67e3..30b1a343f6 100644
--- a/client/jdbc/pom.xml
+++ b/client/jdbc/pom.xml
@@ -21,7 +21,7 @@
com.dremio.client
dremio-client-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-client-jdbc
Client - JDBC Driver
diff --git a/client/pom.xml b/client/pom.xml
index 9981cfd442..8326b2f895 100644
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.client
diff --git a/common/pom.xml b/common/pom.xml
index fa6543d5a4..1a01eeb554 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -23,7 +23,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-common
diff --git a/contrib/hive-exec-shade/pom.xml b/contrib/hive-exec-shade/pom.xml
index 88faa14901..cca74f0d2e 100644
--- a/contrib/hive-exec-shade/pom.xml
+++ b/contrib/hive-exec-shade/pom.xml
@@ -22,7 +22,7 @@
com.dremio.contrib
dremio-contrib-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-hive-exec-shaded
diff --git a/contrib/pom.xml b/contrib/pom.xml
index 913fa7ce3a..7dafdaf7ef 100644
--- a/contrib/pom.xml
+++ b/contrib/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.contrib
diff --git a/contrib/slim-shade/pom.xml b/contrib/slim-shade/pom.xml
index c13f0ad50e..8753eaf0c8 100644
--- a/contrib/slim-shade/pom.xml
+++ b/contrib/slim-shade/pom.xml
@@ -22,7 +22,7 @@
com.dremio.contrib
dremio-contrib-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-slim-shade
diff --git a/contrib/tpch-sample-data/pom.xml b/contrib/tpch-sample-data/pom.xml
index bebf9be7af..ed0858798b 100644
--- a/contrib/tpch-sample-data/pom.xml
+++ b/contrib/tpch-sample-data/pom.xml
@@ -21,7 +21,7 @@
dremio-contrib-parent
com.dremio.contrib
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-tpch-sample-data
diff --git a/dac/backend/pom.xml b/dac/backend/pom.xml
index d88503b1c5..54efc8c651 100644
--- a/dac/backend/pom.xml
+++ b/dac/backend/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-dac-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-dac-backend
diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java
index dc02ee1a0e..26aee1bf78 100644
--- a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java
+++ b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java
@@ -121,7 +121,7 @@ public SchemaMutability getSchemaMutability() {
@Override
public HomeFileSystemStoragePlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) {
- return new HomeFileSystemStoragePlugin(this, context, name, null, pluginIdProvider);
+ return new HomeFileSystemStoragePlugin(this, context, name, pluginIdProvider);
}
public Path getBaseUploadsPath() {
diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java
index af8c012d51..bc966eb0b6 100644
--- a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java
+++ b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java
@@ -66,8 +66,8 @@ public class HomeFileSystemStoragePlugin extends FileSystemPlugin
private final Path stagingDir;
private final Path uploadsDir;
- public HomeFileSystemStoragePlugin(final HomeFileConf config, final SabotContext context, final String name, FileSystemWrapper fs, Provider idProvider) {
- super(config, context, name, fs, idProvider);
+ public HomeFileSystemStoragePlugin(final HomeFileConf config, final SabotContext context, final String name, Provider idProvider) {
+ super(config, context, name, idProvider);
this.stagingDir = new Path(config.getPath(), STAGING + "." + context.getDremioConfig().getThisNode());
this.uploadsDir = new Path(config.getPath(), UPLOADS);
}
@@ -80,7 +80,7 @@ public HomeFileConf getConfig() {
@Override
public void start() throws IOException {
super.start();
- FileSystem fs = getFs();
+ FileSystem fs = getSystemUserFS();
fs.mkdirs(getConfig().getPath(), DEFAULT_PERMISSIONS);
fs.mkdirs(stagingDir, DEFAULT_PERMISSIONS);
fs.mkdirs(uploadsDir, DEFAULT_PERMISSIONS);
@@ -96,7 +96,7 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig
PhysicalDataset physicalDataset = oldConfig == null ? null : oldConfig.getPhysicalDataset();
FormatPluginConfig pluginConfig = null;
try {
- final FileSystemWrapper fs = getFs();
+ final FileSystemWrapper fs = getSystemUserFS();
final FileConfig fileConfig;
if(physicalDataset != null && physicalDataset.getFormatSettings() != null){
@@ -113,7 +113,7 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig
pluginConfig = PhysicalDatasetUtils.toFormatPlugin(fileConfig, Collections.emptyList());
final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(pluginConfig);
return getDataset(datasetPath, oldConfig, formatPlugin, fs, fileConfig,
- retrievalOptions.maxMetadataLeafColumns());
+ retrievalOptions.maxMetadataLeafColumns());
} catch (NamespaceNotFoundException nfe){
FormatPluginConfig formatPluginConfig = null;
if(physicalDataset != null && physicalDataset.getFormatSettings() != null){
@@ -125,9 +125,9 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig
@Override
protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPluginConfig formatPluginConfig,
- DatasetRetrievalOptions retrievalOptions, String user) throws Exception {
+ DatasetRetrievalOptions retrievalOptions, String user) throws Exception {
try{
- final FileSystemWrapper fs = getFs();
+ final FileSystemWrapper fs = getSystemUserFS();
final DatasetConfig datasetConfig = getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath);
if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) {
@@ -137,7 +137,7 @@ protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, D
final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(formatPluginConfig);
return getDataset(datasetPath, oldConfig, formatPlugin, fs,
- datasetConfig.getPhysicalDataset().getFormatSettings(), retrievalOptions.maxMetadataLeafColumns());
+ datasetConfig.getPhysicalDataset().getFormatSettings(), retrievalOptions.maxMetadataLeafColumns());
} catch (NamespaceNotFoundException nfe){
if(formatPluginConfig == null) {
// a home file can only be read from the namespace or using a format options. Without either, it is invalid, return nothing.
@@ -152,10 +152,10 @@ protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, D
public boolean containerExists(NamespaceKey key) {
List folderPath = key.getPathComponents();
try {
- return getFS(SystemUser.SYSTEM_USERNAME).exists(PathUtils.toFSPath(
- ImmutableList.builder()
- .addAll(folderPath.subList(1, folderPath.size()))
- .build()));
+ return getSystemUserFS().exists(PathUtils.toFSPath(
+ ImmutableList.builder()
+ .addAll(folderPath.subList(1, folderPath.size()))
+ .build()));
}catch(IOException e) {
logger.info("IOException while trying to retrieve home files.", e);
return false;
@@ -163,12 +163,12 @@ public boolean containerExists(NamespaceKey key) {
}
private SourceTableDefinition getDataset(
- NamespaceKey datasetPath,
- DatasetConfig oldConfig,
- FormatPlugin formatPlugin,
- FileSystemWrapper fs,
- FileConfig fileConfig,
- int maxLeafColumns
+ NamespaceKey datasetPath,
+ DatasetConfig oldConfig,
+ FormatPlugin formatPlugin,
+ FileSystemWrapper fs,
+ FileConfig fileConfig,
+ int maxLeafColumns
) throws IOException {
final List cachedEntities = Lists.newArrayList();
diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java
index e6d1cae382..37022fe95b 100644
--- a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java
+++ b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java
@@ -55,7 +55,7 @@ public HomeFileTool(SabotContext context, CatalogService catalog) throws Executi
StoragePlugin plugin = catalog.getSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME);
Preconditions.checkNotNull(plugin, "Plugin [%s] not found.", HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME);
HomeFileSystemStoragePlugin homePlugin = (HomeFileSystemStoragePlugin) plugin;
- this.fs = homePlugin.getProcessFs();
+ this.fs = homePlugin.getSystemUserFS();
this.config = homePlugin.getConfig();
this.hostname = context.getDremioConfig().getThisNode();
}
diff --git a/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java b/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java
index 6f39d7f6ed..8589f0d812 100644
--- a/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java
+++ b/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java
@@ -168,7 +168,7 @@ public FileFormat getOrDetectFormat(NamespacePath folderPath, DatasetType expect
private FileFormat detectFileFormat(NamespaceKey key) {
final FileSystemPlugin plugin = getPlugin(key);
- final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName());
+ final FileSystemWrapper fs = plugin.createFS(securityContext.getUserPrincipal().getName());
final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents()));
// for now, existing rudimentary behavior that uses extension detection.
@@ -240,7 +240,7 @@ private static FileFormat asFormat(NamespaceKey key, Path path, boolean isFolder
public JobDataFragment previewData(FileFormat format, NamespacePath namespacePath, boolean useFormatLocation) {
final NamespaceKey key = namespacePath.toNamespaceKey();
final FileSystemPlugin plugin = getPlugin(key);
- final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName());
+ final FileSystemWrapper fs = plugin.createFS(securityContext.getUserPrincipal().getName());
final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents()));
// for now, existing rudimentary behavior that uses extension detection.
@@ -342,7 +342,7 @@ private JobDataFragment getData(FileFormat format, FileSystemPlugin plugin, File
if (first) {
first = false;
container.buildSchema();
- if (container.getSchema().getFieldCount() > maxLeafColumns) {
+ if (container.getSchema().getTotalFieldCount() > maxLeafColumns) {
throw UserException.validationError()
.message("Using datasets with more than %d columns is currently disabled.", maxLeafColumns)
.build(logger);
diff --git a/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java b/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java
index 37dac5a4f3..603dbd5343 100644
--- a/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java
+++ b/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java
@@ -110,7 +110,8 @@ public DatasetVersionMutator(
private DatasetDownloadManager downloadManager() {
final FileSystemPlugin downloadPlugin = catalogService.getSource(DATASET_DOWNLOAD_STORAGE_PLUGIN);
- return new DatasetDownloadManager(jobsService, namespaceService, downloadPlugin.getConfig().getPath(), downloadPlugin.getFs());
+ return new DatasetDownloadManager(jobsService, namespaceService, downloadPlugin.getConfig().getPath(),
+ downloadPlugin.getSystemUserFS());
}
private void validate(DatasetPath path, VirtualDatasetUI ds) {
if (ds.getSqlFieldsList() == null || ds.getSqlFieldsList().isEmpty()) {
diff --git a/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java b/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java
index 2c61216bac..2cad946397 100644
--- a/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java
+++ b/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java
@@ -73,7 +73,7 @@ public SchemaMutability getSchemaMutability() {
@Override
public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) {
- return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider);
+ return new FileSystemPlugin<>(this, context, name, pluginIdProvider);
}
diff --git a/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java b/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java
index 9634938bcc..000b6d797e 100644
--- a/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java
+++ b/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java
@@ -32,8 +32,11 @@
import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import com.dremio.common.exceptions.UserException;
import com.dremio.common.util.FileUtils;
import com.dremio.common.utils.PathUtils;
import com.dremio.dac.explore.model.FileFormatUI;
@@ -73,6 +76,8 @@
* Tests to create, update and execute queries on physical datasets..
*/
public class TestPhysicalDatasets extends BaseTestServer {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
@Before
public void setup() throws Exception {
@@ -294,6 +299,30 @@ public void testSchemaChangeExit() throws Exception {
assertEquals(1, data.getColumns().size());
}
+ @Test
+ public void testLargeJsonFile() throws Exception {
+ final JobsService jobsService = l(JobsService.class);
+ JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder()
+ .setSqlQuery(createQuery("/datasets/wide_table.json"))
+ .setQueryType(QueryType.UI_RUN)
+ .build(), NoOpJobStatusListener.INSTANCE));
+
+ thrown.expect(UserException.class);
+ job.getData().truncate(500);
+ }
+
+ @Test
+ public void testLargeNestedJsonFile() throws Exception {
+ final JobsService jobsService = l(JobsService.class);
+ JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder()
+ .setSqlQuery(createQuery("/datasets/wide_nested_table.json"))
+ .setQueryType(QueryType.UI_RUN)
+ .build(), NoOpJobStatusListener.INSTANCE));
+
+ thrown.expect(UserException.class);
+ job.getData().truncate(500);
+ }
+
@Test
public void testParquetFile() throws Exception {
final JobsService jobsService = l(JobsService.class);
diff --git a/dac/backend/src/test/resources/datasets/wide_nested_table.json b/dac/backend/src/test/resources/datasets/wide_nested_table.json
new file mode 100644
index 0000000000..cd86e7ea2f
--- /dev/null
+++ b/dac/backend/src/test/resources/datasets/wide_nested_table.json
@@ -0,0 +1 @@
+{ "name" : "test_name", "fields": { "column1" : "col", "column2" : "col", "column3" : "col", "column4" : "col", "column5" : "col", "column6" : "col", "column7" : "col", "column8" : "col", "column9" : "col", "column10" : "col", "column11" : "col", "column12" : "col", "column13" : "col", "column14" : "col", "column15" : "col", "column16" : "col", "column17" : "col", "column18" : "col", "column19" : "col", "column20" : "col", "column21" : "col", "column22" : "col", "column23" : "col", "column24" : "col", "column25" : "col", "column26" : "col", "column27" : "col", "column28" : "col", "column29" : "col", "column30" : "col", "column31" : "col", "column32" : "col", "column33" : "col", "column34" : "col", "column35" : "col", "column36" : "col", "column37" : "col", "column38" : "col", "column39" : "col", "column40" : "col", "column41" : "col", "column42" : "col", "column43" : "col", "column44" : "col", "column45" : "col", "column46" : "col", "column47" : "col", "column48" : "col", "column49" : "col", "column50" : "col", "column51" : "col", "column52" : "col", "column53" : "col", "column54" : "col", "column55" : "col", "column56" : "col", "column57" : "col", "column58" : "col", "column59" : "col", "column60" : "col", "column61" : "col", "column62" : "col", "column63" : "col", "column64" : "col", "column65" : "col", "column66" : "col", "column67" : "col", "column68" : "col", "column69" : "col", "column70" : "col", "column71" : "col", "column72" : "col", "column73" : "col", "column74" : "col", "column75" : "col", "column76" : "col", "column77" : "col", "column78" : "col", "column79" : "col", "column80" : "col", "column81" : "col", "column82" : "col", "column83" : "col", "column84" : "col", "column85" : "col", "column86" : "col", "column87" : "col", "column88" : "col", "column89" : "col", "column90" : "col", "column91" : "col", "column92" : "col", "column93" : "col", "column94" : "col", "column95" : "col", "column96" : "col", "column97" : "col", "column98" : "col", "column99" : "col", "column100" : "col" }, "more_fields": { "local_fields": { "column101" : "col", "column102" : "col", "column103" : "col", "column104" : "col", "column105" : "col", "column106" : "col", "column107" : "col", "column108" : "col", "column109" : "col", "column110" : "col", "column111" : "col", "column112" : "col", "column113" : "col", "column114" : "col", "column115" : "col", "column116" : "col", "column117" : "col", "column118" : "col", "column119" : "col", "column120" : "col", "column121" : "col", "column122" : "col", "column123" : "col", "column124" : "col", "column125" : "col", "column126" : "col", "column127" : "col", "column128" : "col", "column129" : "col", "column130" : "col", "column131" : "col", "column132" : "col", "column133" : "col", "column134" : "col", "column135" : "col", "column136" : "col", "column137" : "col", "column138" : "col", "column139" : "col", "column140" : "col", "column141" : "col", "column142" : "col", "column143" : "col", "column144" : "col", "column145" : "col", "column146" : "col", "column147" : "col", "column148" : "col", "column149" : "col", "column150" : "col", "column151" : "col", "column152" : "col", "column153" : "col", "column154" : "col", "column155" : "col", "column156" : "col", "column157" : "col", "column158" : "col", "column159" : "col", "column160" : "col", "column161" : "col", "column162" : "col", "column163" : "col", "column164" : "col", "column165" : "col", "column166" : "col", "column167" : "col", "column168" : "col", "column169" : "col", "column170" : "col", "column171" : "col", "column172" : "col", "column173" : "col", "column174" : "col", "column175" : "col", "column176" : "col", "column177" : "col", "column178" : "col", "column179" : "col", "column180" : "col", "column181" : "col", "column182" : "col", "column183" : "col", "column184" : "col", "column185" : "col", "column186" : "col", "column187" : "col", "column188" : "col", "column189" : "col", "column190" : "col", "column191" : "col", "column192" : "col", "column193" : "col", "column194" : "col", "column195" : "col", "column196" : "col", "column197" : "col", "column198" : "col", "column199" : "col", "column200" : "col", "column201" : "col", "column202" : "col", "column203" : "col", "column204" : "col", "column205" : "col", "column206" : "col", "column207" : "col", "column208" : "col", "column209" : "col", "column210" : "col", "column211" : "col", "column212" : "col", "column213" : "col", "column214" : "col", "column215" : "col", "column216" : "col", "column217" : "col", "column218" : "col", "column219" : "col", "column220" : "col", "column221" : "col", "column222" : "col", "column223" : "col", "column224" : "col", "column225" : "col", "column226" : "col", "column227" : "col", "column228" : "col", "column229" : "col", "column230" : "col", "column231" : "col", "column232" : "col", "column233" : "col", "column234" : "col", "column235" : "col", "column236" : "col", "column237" : "col", "column238" : "col", "column239" : "col", "column240" : "col", "column241" : "col", "column242" : "col", "column243" : "col", "column244" : "col", "column245" : "col", "column246" : "col", "column247" : "col", "column248" : "col", "column249" : "col", "column250" : "col", "column251" : "col", "column252" : "col", "column253" : "col", "column254" : "col", "column255" : "col", "column256" : "col", "column257" : "col", "column258" : "col", "column259" : "col", "column260" : "col", "column261" : "col", "column262" : "col", "column263" : "col", "column264" : "col", "column265" : "col", "column266" : "col", "column267" : "col", "column268" : "col", "column269" : "col", "column270" : "col", "column271" : "col", "column272" : "col", "column273" : "col", "column274" : "col", "column275" : "col", "column276" : "col", "column277" : "col", "column278" : "col", "column279" : "col", "column280" : "col", "column281" : "col", "column282" : "col", "column283" : "col", "column284" : "col", "column285" : "col", "column286" : "col", "column287" : "col", "column288" : "col", "column289" : "col", "column290" : "col", "column291" : "col", "column292" : "col", "column293" : "col", "column294" : "col", "column295" : "col", "column296" : "col", "column297" : "col", "column298" : "col", "column299" : "col", "column300" : "col", "column301" : "col", "column302" : "col", "column303" : "col", "column304" : "col", "column305" : "col", "column306" : "col", "column307" : "col", "column308" : "col", "column309" : "col", "column310" : "col", "column311" : "col", "column312" : "col", "column313" : "col", "column314" : "col", "column315" : "col", "column316" : "col", "column317" : "col", "column318" : "col", "column319" : "col", "column320" : "col", "column321" : "col", "column322" : "col", "column323" : "col", "column324" : "col", "column325" : "col", "column326" : "col", "column327" : "col", "column328" : "col", "column329" : "col", "column330" : "col", "column331" : "col", "column332" : "col", "column333" : "col", "column334" : "col", "column335" : "col", "column336" : "col", "column337" : "col", "column338" : "col", "column339" : "col", "column340" : "col", "column341" : "col", "column342" : "col", "column343" : "col", "column344" : "col", "column345" : "col", "column346" : "col", "column347" : "col", "column348" : "col", "column349" : "col", "column350" : "col", "column351" : "col", "column352" : "col", "column353" : "col", "column354" : "col", "column355" : "col", "column356" : "col", "column357" : "col", "column358" : "col", "column359" : "col", "column360" : "col", "column361" : "col", "column362" : "col", "column363" : "col", "column364" : "col", "column365" : "col", "column366" : "col", "column367" : "col", "column368" : "col", "column369" : "col", "column370" : "col", "column371" : "col", "column372" : "col", "column373" : "col", "column374" : "col", "column375" : "col", "column376" : "col", "column377" : "col", "column378" : "col", "column379" : "col", "column380" : "col", "column381" : "col", "column382" : "col", "column383" : "col", "column384" : "col", "column385" : "col", "column386" : "col", "column387" : "col", "column388" : "col", "column389" : "col", "column390" : "col", "column391" : "col", "column392" : "col", "column393" : "col", "column394" : "col", "column395" : "col", "column396" : "col", "column397" : "col", "column398" : "col", "column399" : "col" } }, "column400" : "col", "column401" : "col", "column402" : "col", "column403" : "col", "column404" : "col", "column405" : "col", "column406" : "col", "column407" : "col", "column408" : "col", "column409" : "col", "column410" : "col", "column411" : "col", "column412" : "col", "column413" : "col", "column414" : "col", "column415" : "col", "column416" : "col", "column417" : "col", "column418" : "col", "column419" : "col", "column420" : "col", "column421" : "col", "column422" : "col", "column423" : "col", "column424" : "col", "column425" : "col", "column426" : "col", "column427" : "col", "column428" : "col", "column429" : "col", "column430" : "col", "column431" : "col", "column432" : "col", "column433" : "col", "column434" : "col", "column435" : "col", "column436" : "col", "column437" : "col", "column438" : "col", "column439" : "col", "column440" : "col", "column441" : "col", "column442" : "col", "column443" : "col", "column444" : "col", "column445" : "col", "column446" : "col", "column447" : "col", "column448" : "col", "column449" : "col", "column450" : "col", "column451" : "col", "column452" : "col", "column453" : "col", "column454" : "col", "column455" : "col", "column456" : "col", "column457" : "col", "column458" : "col", "column459" : "col", "column460" : "col", "column461" : "col", "column462" : "col", "column463" : "col", "column464" : "col", "column465" : "col", "column466" : "col", "column467" : "col", "column468" : "col", "column469" : "col", "column470" : "col", "column471" : "col", "column472" : "col", "column473" : "col", "column474" : "col", "column475" : "col", "column476" : "col", "column477" : "col", "column478" : "col", "column479" : "col", "column480" : "col", "column481" : "col", "column482" : "col", "column483" : "col", "column484" : "col", "column485" : "col", "column486" : "col", "column487" : "col", "column488" : "col", "column489" : "col", "column490" : "col", "column491" : "col", "column492" : "col", "column493" : "col", "column494" : "col", "column495" : "col", "column496" : "col", "column497" : "col", "column498" : "col", "column499" : "col", "column500" : "col", "column501" : "col", "column502" : "col", "column503" : "col", "column504" : "col", "column505" : "col", "column506" : "col", "column507" : "col", "column508" : "col", "column509" : "col", "column510" : "col", "column511" : "col", "column512" : "col", "column513" : "col", "column514" : "col", "column515" : "col", "column516" : "col", "column517" : "col", "column518" : "col", "column519" : "col", "column520" : "col", "column521" : "col", "column522" : "col", "column523" : "col", "column524" : "col", "column525" : "col", "column526" : "col", "column527" : "col", "column528" : "col", "column529" : "col", "column530" : "col", "column531" : "col", "column532" : "col", "column533" : "col", "column534" : "col", "column535" : "col", "column536" : "col", "column537" : "col", "column538" : "col", "column539" : "col", "column540" : "col", "column541" : "col", "column542" : "col", "column543" : "col", "column544" : "col", "column545" : "col", "column546" : "col", "column547" : "col", "column548" : "col", "column549" : "col", "column550" : "col", "column551" : "col", "column552" : "col", "column553" : "col", "column554" : "col", "column555" : "col", "column556" : "col", "column557" : "col", "column558" : "col", "column559" : "col", "column560" : "col", "column561" : "col", "column562" : "col", "column563" : "col", "column564" : "col", "column565" : "col", "column566" : "col", "column567" : "col", "column568" : "col", "column569" : "col", "column570" : "col", "column571" : "col", "column572" : "col", "column573" : "col", "column574" : "col", "column575" : "col", "column576" : "col", "column577" : "col", "column578" : "col", "column579" : "col", "column580" : "col", "column581" : "col", "column582" : "col", "column583" : "col", "column584" : "col", "column585" : "col", "column586" : "col", "column587" : "col", "column588" : "col", "column589" : "col", "column590" : "col", "column591" : "col", "column592" : "col", "column593" : "col", "column594" : "col", "column595" : "col", "column596" : "col", "column597" : "col", "column598" : "col", "column599" : "col", "column600" : "col", "column601" : "col", "column602" : "col", "column603" : "col", "column604" : "col", "column605" : "col", "column606" : "col", "column607" : "col", "column608" : "col", "column609" : "col", "column610" : "col", "column611" : "col", "column612" : "col", "column613" : "col", "column614" : "col", "column615" : "col", "column616" : "col", "column617" : "col", "column618" : "col", "column619" : "col", "column620" : "col", "column621" : "col", "column622" : "col", "column623" : "col", "column624" : "col", "column625" : "col", "column626" : "col", "column627" : "col", "column628" : "col", "column629" : "col", "column630" : "col", "column631" : "col", "column632" : "col", "column633" : "col", "column634" : "col", "column635" : "col", "column636" : "col", "column637" : "col", "column638" : "col", "column639" : "col", "column640" : "col", "column641" : "col", "column642" : "col", "column643" : "col", "column644" : "col", "column645" : "col", "column646" : "col", "column647" : "col", "column648" : "col", "column649" : "col", "column650" : "col", "column651" : "col", "column652" : "col", "column653" : "col", "column654" : "col", "column655" : "col", "column656" : "col", "column657" : "col", "column658" : "col", "column659" : "col", "column660" : "col", "column661" : "col", "column662" : "col", "column663" : "col", "column664" : "col", "column665" : "col", "column666" : "col", "column667" : "col", "column668" : "col", "column669" : "col", "column670" : "col", "column671" : "col", "column672" : "col", "column673" : "col", "column674" : "col", "column675" : "col", "column676" : "col", "column677" : "col", "column678" : "col", "column679" : "col", "column680" : "col", "column681" : "col", "column682" : "col", "column683" : "col", "column684" : "col", "column685" : "col", "column686" : "col", "column687" : "col", "column688" : "col", "column689" : "col", "column690" : "col", "column691" : "col", "column692" : "col", "column693" : "col", "column694" : "col", "column695" : "col", "column696" : "col", "column697" : "col", "column698" : "col", "column699" : "col", "column700" : "col", "column701" : "col", "column702" : "col", "column703" : "col", "column704" : "col", "column705" : "col", "column706" : "col", "column707" : "col", "column708" : "col", "column709" : "col", "column710" : "col", "column711" : "col", "column712" : "col", "column713" : "col", "column714" : "col", "column715" : "col", "column716" : "col", "column717" : "col", "column718" : "col", "column719" : "col", "column720" : "col", "column721" : "col", "column722" : "col", "column723" : "col", "column724" : "col", "column725" : "col", "column726" : "col", "column727" : "col", "column728" : "col", "column729" : "col", "column730" : "col", "column731" : "col", "column732" : "col", "column733" : "col", "column734" : "col", "column735" : "col", "column736" : "col", "column737" : "col", "column738" : "col", "column739" : "col", "column740" : "col", "column741" : "col", "column742" : "col", "column743" : "col", "column744" : "col", "column745" : "col", "column746" : "col", "column747" : "col", "column748" : "col", "column749" : "col", "column750" : "col", "column751" : "col", "column752" : "col", "column753" : "col", "column754" : "col", "column755" : "col", "column756" : "col", "column757" : "col", "column758" : "col", "column759" : "col", "column760" : "col", "column761" : "col", "column762" : "col", "column763" : "col", "column764" : "col", "column765" : "col", "column766" : "col", "column767" : "col", "column768" : "col", "column769" : "col", "column770" : "col", "column771" : "col", "column772" : "col", "column773" : "col", "column774" : "col", "column775" : "col", "column776" : "col", "column777" : "col", "column778" : "col", "column779" : "col", "column780" : "col", "column781" : "col", "column782" : "col", "column783" : "col", "column784" : "col", "column785" : "col", "column786" : "col", "column787" : "col", "column788" : "col", "column789" : "col", "column790" : "col", "column791" : "col", "column792" : "col", "column793" : "col", "column794" : "col", "column795" : "col", "column796" : "col", "column797" : "col", "column798" : "col", "column799" : "col", "column800" : "col", "column801" : "col", "column802" : "col", "column803" : "col", "column804" : "col", "column805" : "col", "column806" : "col", "column807" : "col", "column808" : "col", "column809" : "col", "column810" : "col" }
diff --git a/dac/backend/src/test/resources/datasets/wide_table.json b/dac/backend/src/test/resources/datasets/wide_table.json
new file mode 100644
index 0000000000..0df384a674
--- /dev/null
+++ b/dac/backend/src/test/resources/datasets/wide_table.json
@@ -0,0 +1 @@
+{ "column1" : "col", "column2" : "col", "column3" : "col", "column4" : "col", "column5" : "col", "column6" : "col", "column7" : "col", "column8" : "col", "column9" : "col", "column10" : "col", "column11" : "col", "column12" : "col", "column13" : "col", "column14" : "col", "column15" : "col", "column16" : "col", "column17" : "col", "column18" : "col", "column19" : "col", "column20" : "col", "column21" : "col", "column22" : "col", "column23" : "col", "column24" : "col", "column25" : "col", "column26" : "col", "column27" : "col", "column28" : "col", "column29" : "col", "column30" : "col", "column31" : "col", "column32" : "col", "column33" : "col", "column34" : "col", "column35" : "col", "column36" : "col", "column37" : "col", "column38" : "col", "column39" : "col", "column40" : "col", "column41" : "col", "column42" : "col", "column43" : "col", "column44" : "col", "column45" : "col", "column46" : "col", "column47" : "col", "column48" : "col", "column49" : "col", "column50" : "col", "column51" : "col", "column52" : "col", "column53" : "col", "column54" : "col", "column55" : "col", "column56" : "col", "column57" : "col", "column58" : "col", "column59" : "col", "column60" : "col", "column61" : "col", "column62" : "col", "column63" : "col", "column64" : "col", "column65" : "col", "column66" : "col", "column67" : "col", "column68" : "col", "column69" : "col", "column70" : "col", "column71" : "col", "column72" : "col", "column73" : "col", "column74" : "col", "column75" : "col", "column76" : "col", "column77" : "col", "column78" : "col", "column79" : "col", "column80" : "col", "column81" : "col", "column82" : "col", "column83" : "col", "column84" : "col", "column85" : "col", "column86" : "col", "column87" : "col", "column88" : "col", "column89" : "col", "column90" : "col", "column91" : "col", "column92" : "col", "column93" : "col", "column94" : "col", "column95" : "col", "column96" : "col", "column97" : "col", "column98" : "col", "column99" : "col", "column100" : "col", "column101" : "col", "column102" : "col", "column103" : "col", "column104" : "col", "column105" : "col", "column106" : "col", "column107" : "col", "column108" : "col", "column109" : "col", "column110" : "col", "column111" : "col", "column112" : "col", "column113" : "col", "column114" : "col", "column115" : "col", "column116" : "col", "column117" : "col", "column118" : "col", "column119" : "col", "column120" : "col", "column121" : "col", "column122" : "col", "column123" : "col", "column124" : "col", "column125" : "col", "column126" : "col", "column127" : "col", "column128" : "col", "column129" : "col", "column130" : "col", "column131" : "col", "column132" : "col", "column133" : "col", "column134" : "col", "column135" : "col", "column136" : "col", "column137" : "col", "column138" : "col", "column139" : "col", "column140" : "col", "column141" : "col", "column142" : "col", "column143" : "col", "column144" : "col", "column145" : "col", "column146" : "col", "column147" : "col", "column148" : "col", "column149" : "col", "column150" : "col", "column151" : "col", "column152" : "col", "column153" : "col", "column154" : "col", "column155" : "col", "column156" : "col", "column157" : "col", "column158" : "col", "column159" : "col", "column160" : "col", "column161" : "col", "column162" : "col", "column163" : "col", "column164" : "col", "column165" : "col", "column166" : "col", "column167" : "col", "column168" : "col", "column169" : "col", "column170" : "col", "column171" : "col", "column172" : "col", "column173" : "col", "column174" : "col", "column175" : "col", "column176" : "col", "column177" : "col", "column178" : "col", "column179" : "col", "column180" : "col", "column181" : "col", "column182" : "col", "column183" : "col", "column184" : "col", "column185" : "col", "column186" : "col", "column187" : "col", "column188" : "col", "column189" : "col", "column190" : "col", "column191" : "col", "column192" : "col", "column193" : "col", "column194" : "col", "column195" : "col", "column196" : "col", "column197" : "col", "column198" : "col", "column199" : "col", "column200" : "col", "column201" : "col", "column202" : "col", "column203" : "col", "column204" : "col", "column205" : "col", "column206" : "col", "column207" : "col", "column208" : "col", "column209" : "col", "column210" : "col", "column211" : "col", "column212" : "col", "column213" : "col", "column214" : "col", "column215" : "col", "column216" : "col", "column217" : "col", "column218" : "col", "column219" : "col", "column220" : "col", "column221" : "col", "column222" : "col", "column223" : "col", "column224" : "col", "column225" : "col", "column226" : "col", "column227" : "col", "column228" : "col", "column229" : "col", "column230" : "col", "column231" : "col", "column232" : "col", "column233" : "col", "column234" : "col", "column235" : "col", "column236" : "col", "column237" : "col", "column238" : "col", "column239" : "col", "column240" : "col", "column241" : "col", "column242" : "col", "column243" : "col", "column244" : "col", "column245" : "col", "column246" : "col", "column247" : "col", "column248" : "col", "column249" : "col", "column250" : "col", "column251" : "col", "column252" : "col", "column253" : "col", "column254" : "col", "column255" : "col", "column256" : "col", "column257" : "col", "column258" : "col", "column259" : "col", "column260" : "col", "column261" : "col", "column262" : "col", "column263" : "col", "column264" : "col", "column265" : "col", "column266" : "col", "column267" : "col", "column268" : "col", "column269" : "col", "column270" : "col", "column271" : "col", "column272" : "col", "column273" : "col", "column274" : "col", "column275" : "col", "column276" : "col", "column277" : "col", "column278" : "col", "column279" : "col", "column280" : "col", "column281" : "col", "column282" : "col", "column283" : "col", "column284" : "col", "column285" : "col", "column286" : "col", "column287" : "col", "column288" : "col", "column289" : "col", "column290" : "col", "column291" : "col", "column292" : "col", "column293" : "col", "column294" : "col", "column295" : "col", "column296" : "col", "column297" : "col", "column298" : "col", "column299" : "col", "column300" : "col", "column301" : "col", "column302" : "col", "column303" : "col", "column304" : "col", "column305" : "col", "column306" : "col", "column307" : "col", "column308" : "col", "column309" : "col", "column310" : "col", "column311" : "col", "column312" : "col", "column313" : "col", "column314" : "col", "column315" : "col", "column316" : "col", "column317" : "col", "column318" : "col", "column319" : "col", "column320" : "col", "column321" : "col", "column322" : "col", "column323" : "col", "column324" : "col", "column325" : "col", "column326" : "col", "column327" : "col", "column328" : "col", "column329" : "col", "column330" : "col", "column331" : "col", "column332" : "col", "column333" : "col", "column334" : "col", "column335" : "col", "column336" : "col", "column337" : "col", "column338" : "col", "column339" : "col", "column340" : "col", "column341" : "col", "column342" : "col", "column343" : "col", "column344" : "col", "column345" : "col", "column346" : "col", "column347" : "col", "column348" : "col", "column349" : "col", "column350" : "col", "column351" : "col", "column352" : "col", "column353" : "col", "column354" : "col", "column355" : "col", "column356" : "col", "column357" : "col", "column358" : "col", "column359" : "col", "column360" : "col", "column361" : "col", "column362" : "col", "column363" : "col", "column364" : "col", "column365" : "col", "column366" : "col", "column367" : "col", "column368" : "col", "column369" : "col", "column370" : "col", "column371" : "col", "column372" : "col", "column373" : "col", "column374" : "col", "column375" : "col", "column376" : "col", "column377" : "col", "column378" : "col", "column379" : "col", "column380" : "col", "column381" : "col", "column382" : "col", "column383" : "col", "column384" : "col", "column385" : "col", "column386" : "col", "column387" : "col", "column388" : "col", "column389" : "col", "column390" : "col", "column391" : "col", "column392" : "col", "column393" : "col", "column394" : "col", "column395" : "col", "column396" : "col", "column397" : "col", "column398" : "col", "column399" : "col", "column400" : "col", "column401" : "col", "column402" : "col", "column403" : "col", "column404" : "col", "column405" : "col", "column406" : "col", "column407" : "col", "column408" : "col", "column409" : "col", "column410" : "col", "column411" : "col", "column412" : "col", "column413" : "col", "column414" : "col", "column415" : "col", "column416" : "col", "column417" : "col", "column418" : "col", "column419" : "col", "column420" : "col", "column421" : "col", "column422" : "col", "column423" : "col", "column424" : "col", "column425" : "col", "column426" : "col", "column427" : "col", "column428" : "col", "column429" : "col", "column430" : "col", "column431" : "col", "column432" : "col", "column433" : "col", "column434" : "col", "column435" : "col", "column436" : "col", "column437" : "col", "column438" : "col", "column439" : "col", "column440" : "col", "column441" : "col", "column442" : "col", "column443" : "col", "column444" : "col", "column445" : "col", "column446" : "col", "column447" : "col", "column448" : "col", "column449" : "col", "column450" : "col", "column451" : "col", "column452" : "col", "column453" : "col", "column454" : "col", "column455" : "col", "column456" : "col", "column457" : "col", "column458" : "col", "column459" : "col", "column460" : "col", "column461" : "col", "column462" : "col", "column463" : "col", "column464" : "col", "column465" : "col", "column466" : "col", "column467" : "col", "column468" : "col", "column469" : "col", "column470" : "col", "column471" : "col", "column472" : "col", "column473" : "col", "column474" : "col", "column475" : "col", "column476" : "col", "column477" : "col", "column478" : "col", "column479" : "col", "column480" : "col", "column481" : "col", "column482" : "col", "column483" : "col", "column484" : "col", "column485" : "col", "column486" : "col", "column487" : "col", "column488" : "col", "column489" : "col", "column490" : "col", "column491" : "col", "column492" : "col", "column493" : "col", "column494" : "col", "column495" : "col", "column496" : "col", "column497" : "col", "column498" : "col", "column499" : "col", "column500" : "col", "column501" : "col", "column502" : "col", "column503" : "col", "column504" : "col", "column505" : "col", "column506" : "col", "column507" : "col", "column508" : "col", "column509" : "col", "column510" : "col", "column511" : "col", "column512" : "col", "column513" : "col", "column514" : "col", "column515" : "col", "column516" : "col", "column517" : "col", "column518" : "col", "column519" : "col", "column520" : "col", "column521" : "col", "column522" : "col", "column523" : "col", "column524" : "col", "column525" : "col", "column526" : "col", "column527" : "col", "column528" : "col", "column529" : "col", "column530" : "col", "column531" : "col", "column532" : "col", "column533" : "col", "column534" : "col", "column535" : "col", "column536" : "col", "column537" : "col", "column538" : "col", "column539" : "col", "column540" : "col", "column541" : "col", "column542" : "col", "column543" : "col", "column544" : "col", "column545" : "col", "column546" : "col", "column547" : "col", "column548" : "col", "column549" : "col", "column550" : "col", "column551" : "col", "column552" : "col", "column553" : "col", "column554" : "col", "column555" : "col", "column556" : "col", "column557" : "col", "column558" : "col", "column559" : "col", "column560" : "col", "column561" : "col", "column562" : "col", "column563" : "col", "column564" : "col", "column565" : "col", "column566" : "col", "column567" : "col", "column568" : "col", "column569" : "col", "column570" : "col", "column571" : "col", "column572" : "col", "column573" : "col", "column574" : "col", "column575" : "col", "column576" : "col", "column577" : "col", "column578" : "col", "column579" : "col", "column580" : "col", "column581" : "col", "column582" : "col", "column583" : "col", "column584" : "col", "column585" : "col", "column586" : "col", "column587" : "col", "column588" : "col", "column589" : "col", "column590" : "col", "column591" : "col", "column592" : "col", "column593" : "col", "column594" : "col", "column595" : "col", "column596" : "col", "column597" : "col", "column598" : "col", "column599" : "col", "column600" : "col", "column601" : "col", "column602" : "col", "column603" : "col", "column604" : "col", "column605" : "col", "column606" : "col", "column607" : "col", "column608" : "col", "column609" : "col", "column610" : "col", "column611" : "col", "column612" : "col", "column613" : "col", "column614" : "col", "column615" : "col", "column616" : "col", "column617" : "col", "column618" : "col", "column619" : "col", "column620" : "col", "column621" : "col", "column622" : "col", "column623" : "col", "column624" : "col", "column625" : "col", "column626" : "col", "column627" : "col", "column628" : "col", "column629" : "col", "column630" : "col", "column631" : "col", "column632" : "col", "column633" : "col", "column634" : "col", "column635" : "col", "column636" : "col", "column637" : "col", "column638" : "col", "column639" : "col", "column640" : "col", "column641" : "col", "column642" : "col", "column643" : "col", "column644" : "col", "column645" : "col", "column646" : "col", "column647" : "col", "column648" : "col", "column649" : "col", "column650" : "col", "column651" : "col", "column652" : "col", "column653" : "col", "column654" : "col", "column655" : "col", "column656" : "col", "column657" : "col", "column658" : "col", "column659" : "col", "column660" : "col", "column661" : "col", "column662" : "col", "column663" : "col", "column664" : "col", "column665" : "col", "column666" : "col", "column667" : "col", "column668" : "col", "column669" : "col", "column670" : "col", "column671" : "col", "column672" : "col", "column673" : "col", "column674" : "col", "column675" : "col", "column676" : "col", "column677" : "col", "column678" : "col", "column679" : "col", "column680" : "col", "column681" : "col", "column682" : "col", "column683" : "col", "column684" : "col", "column685" : "col", "column686" : "col", "column687" : "col", "column688" : "col", "column689" : "col", "column690" : "col", "column691" : "col", "column692" : "col", "column693" : "col", "column694" : "col", "column695" : "col", "column696" : "col", "column697" : "col", "column698" : "col", "column699" : "col", "column700" : "col", "column701" : "col", "column702" : "col", "column703" : "col", "column704" : "col", "column705" : "col", "column706" : "col", "column707" : "col", "column708" : "col", "column709" : "col", "column710" : "col", "column711" : "col", "column712" : "col", "column713" : "col", "column714" : "col", "column715" : "col", "column716" : "col", "column717" : "col", "column718" : "col", "column719" : "col", "column720" : "col", "column721" : "col", "column722" : "col", "column723" : "col", "column724" : "col", "column725" : "col", "column726" : "col", "column727" : "col", "column728" : "col", "column729" : "col", "column730" : "col", "column731" : "col", "column732" : "col", "column733" : "col", "column734" : "col", "column735" : "col", "column736" : "col", "column737" : "col", "column738" : "col", "column739" : "col", "column740" : "col", "column741" : "col", "column742" : "col", "column743" : "col", "column744" : "col", "column745" : "col", "column746" : "col", "column747" : "col", "column748" : "col", "column749" : "col", "column750" : "col", "column751" : "col", "column752" : "col", "column753" : "col", "column754" : "col", "column755" : "col", "column756" : "col", "column757" : "col", "column758" : "col", "column759" : "col", "column760" : "col", "column761" : "col", "column762" : "col", "column763" : "col", "column764" : "col", "column765" : "col", "column766" : "col", "column767" : "col", "column768" : "col", "column769" : "col", "column770" : "col", "column771" : "col", "column772" : "col", "column773" : "col", "column774" : "col", "column775" : "col", "column776" : "col", "column777" : "col", "column778" : "col", "column779" : "col", "column780" : "col", "column781" : "col", "column782" : "col", "column783" : "col", "column784" : "col", "column785" : "col", "column786" : "col", "column787" : "col", "column788" : "col", "column789" : "col", "column790" : "col", "column791" : "col", "column792" : "col", "column793" : "col", "column794" : "col", "column795" : "col", "column796" : "col", "column797" : "col", "column798" : "col", "column799" : "col", "column800" : "col", "column801" : "col", "column802" : "col", "column803" : "col", "column804" : "col", "column805" : "col", "column806" : "col", "column807" : "col", "column808" : "col", "column809" : "col", "column810" : "col" }
diff --git a/dac/common/pom.xml b/dac/common/pom.xml
index 57f6b9070a..b7f5f1f4b6 100644
--- a/dac/common/pom.xml
+++ b/dac/common/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-dac-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-dac-common
diff --git a/dac/daemon/pom.xml b/dac/daemon/pom.xml
index 7b81dd20b2..d743280e1a 100644
--- a/dac/daemon/pom.xml
+++ b/dac/daemon/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-dac-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-dac-daemon
diff --git a/dac/pom.xml b/dac/pom.xml
index b791a598ba..d6518ca090 100644
--- a/dac/pom.xml
+++ b/dac/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-dac-parent
diff --git a/dac/ui/pom.xml b/dac/ui/pom.xml
index 97bafae1a3..0603ed70d7 100644
--- a/dac/ui/pom.xml
+++ b/dac/ui/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-dac-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-dac-ui
diff --git a/distribution/jdbc-driver/pom.xml b/distribution/jdbc-driver/pom.xml
index 67ec216cc4..25e875cf07 100644
--- a/distribution/jdbc-driver/pom.xml
+++ b/distribution/jdbc-driver/pom.xml
@@ -21,7 +21,7 @@
com.dremio.distribution
dremio-distribution-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-jdbc-driver
diff --git a/distribution/pom.xml b/distribution/pom.xml
index c23052a5d4..32499fc545 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.distribution
diff --git a/distribution/resources/pom.xml b/distribution/resources/pom.xml
index d728a4f829..de126de9d5 100644
--- a/distribution/resources/pom.xml
+++ b/distribution/resources/pom.xml
@@ -22,7 +22,7 @@
com.dremio.distribution
dremio-distribution-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-distribution-resources
diff --git a/distribution/resources/src/main/resources/bin/dremio b/distribution/resources/src/main/resources/bin/dremio
index cf358f4c85..f065d06b32 100755
--- a/distribution/resources/src/main/resources/bin/dremio
+++ b/distribution/resources/src/main/resources/bin/dremio
@@ -55,6 +55,15 @@ shift
command=dremio
+print_warn() {
+ echo "#######################################################################"
+ echo "WARNING:"
+ echo ""
+ echo -e "$1"
+ echo ""
+ echo "#######################################################################"
+}
+
wait_for_process_end() {
pidKilled=$1
commandName=$2
@@ -188,13 +197,22 @@ DREMIO_NICENESS="${DREMIO_NICENESS:-0}"
thiscmd=$0
args=$@
+ULIMIT=65536
+if [ "$(ulimit -n)" -lt "$ULIMIT" ]; then
+ print_warn "\
+Please increase the maximum number of file descriptors to at least $ULIMIT.\n\n\
+Dremio utilizes a large number of file descriptors.\n\
+The maximum number of file descriptors, currently set at $(ulimit -n), is not sufficient."
+fi
+
case $startStopStatus in
(start)
check_before_start
- echo starting $command, logging to $logout
+ echo Starting $command, logging to $logout
nohup $thiscmd internal_start $args < /dev/null >> ${logout} 2>&1 &
sleep 1;
+ echo Check the log file to make sure $command started successfully and did not run into any issues.
;;
(start-fg)
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
index 357d98d5f8..ac50478056 100644
--- a/distribution/server/pom.xml
+++ b/distribution/server/pom.xml
@@ -22,7 +22,7 @@
com.dremio.distribution
dremio-distribution-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-distribution
diff --git a/plugins/elasticsearch/pom.xml b/plugins/elasticsearch/pom.xml
index bc5156c8dc..d9baf6e11f 100644
--- a/plugins/elasticsearch/pom.xml
+++ b/plugins/elasticsearch/pom.xml
@@ -26,7 +26,7 @@
dremio-plugin-parent
com.dremio.plugin
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-elasticsearch-plugin
diff --git a/plugins/hbase/pom.xml b/plugins/hbase/pom.xml
index 236cfec2df..561dc565ba 100644
--- a/plugins/hbase/pom.xml
+++ b/plugins/hbase/pom.xml
@@ -22,7 +22,7 @@
com.dremio.plugin
dremio-plugin-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-hbase-plugin
diff --git a/plugins/hive/pom.xml b/plugins/hive/pom.xml
index b32f395b80..e27e17afe0 100644
--- a/plugins/hive/pom.xml
+++ b/plugins/hive/pom.xml
@@ -22,7 +22,7 @@
com.dremio.plugin
dremio-plugin-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-hive-plugin
diff --git a/plugins/jdbc/pom.xml b/plugins/jdbc/pom.xml
index 1cd3f22bf4..81b826e084 100755
--- a/plugins/jdbc/pom.xml
+++ b/plugins/jdbc/pom.xml
@@ -22,7 +22,7 @@
com.dremio.plugin
dremio-plugin-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-jdbc-plugin
diff --git a/plugins/mongo/pom.xml b/plugins/mongo/pom.xml
index 505ec208f3..8fd891d5bf 100644
--- a/plugins/mongo/pom.xml
+++ b/plugins/mongo/pom.xml
@@ -26,7 +26,7 @@
dremio-plugin-parent
com.dremio.plugin
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-mongo-plugin
diff --git a/plugins/pdfs/pom.xml b/plugins/pdfs/pom.xml
index 2c00e1f6ab..98817fe656 100644
--- a/plugins/pdfs/pom.xml
+++ b/plugins/pdfs/pom.xml
@@ -25,7 +25,7 @@
dremio-plugin-parent
com.dremio.plugin
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-pdfs-plugin
diff --git a/plugins/pdfs/src/main/java/com/dremio/exec/store/dfs/PDFSConf.java b/plugins/pdfs/src/main/java/com/dremio/exec/store/dfs/PDFSConf.java
index 2628d90923..c019bdfb8c 100644
--- a/plugins/pdfs/src/main/java/com/dremio/exec/store/dfs/PDFSConf.java
+++ b/plugins/pdfs/src/main/java/com/dremio/exec/store/dfs/PDFSConf.java
@@ -65,7 +65,7 @@ public SchemaMutability getSchemaMutability() {
@Override
public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) {
- return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider);
+ return new FileSystemPlugin<>(this, context, name, pluginIdProvider);
}
}
diff --git a/plugins/pom.xml b/plugins/pom.xml
index b89eb9c78b..892420efe6 100644
--- a/plugins/pom.xml
+++ b/plugins/pom.xml
@@ -24,7 +24,7 @@
dremio-root
com.dremio
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.plugin
diff --git a/plugins/s3/pom.xml b/plugins/s3/pom.xml
index b50395aac5..eacc55f5b9 100644
--- a/plugins/s3/pom.xml
+++ b/plugins/s3/pom.xml
@@ -26,7 +26,7 @@
dremio-plugin-parent
com.dremio.plugin
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-s3-plugin
diff --git a/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeConf.java b/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeConf.java
index 13f870b622..b7e99d2557 100644
--- a/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeConf.java
+++ b/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeConf.java
@@ -111,7 +111,7 @@ public AzureDataLakeStoragePlugin newPlugin(SabotContext context, String name, P
Preconditions.checkNotNull(accountName, "Account name must be set.");
Preconditions.checkNotNull(clientId, "Client ID must be set.");
Preconditions.checkNotNull(mode, "Authentication mode must be set.");
- return new AzureDataLakeStoragePlugin(this, context, name, null, pluginIdProvider);
+ return new AzureDataLakeStoragePlugin(this, context, name, pluginIdProvider);
}
@Override
diff --git a/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeStoragePlugin.java b/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeStoragePlugin.java
index e403cd0f0a..4222b99fa0 100644
--- a/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeStoragePlugin.java
+++ b/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeStoragePlugin.java
@@ -24,7 +24,6 @@
import com.dremio.exec.catalog.conf.Property;
import com.dremio.exec.server.SabotContext;
import com.dremio.exec.store.dfs.FileSystemPlugin;
-import com.dremio.exec.store.dfs.FileSystemWrapper;
/**
* Storage plugin for Microsoft Azure Data Lake
@@ -41,9 +40,9 @@ public class AzureDataLakeStoragePlugin extends FileSystemPlugin idProvider) {
- super(config, context, name, fs, idProvider);
+ super(config, context, name, idProvider);
}
@Override
diff --git a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java
index 10d97954b2..5ba8ace484 100644
--- a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java
+++ b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java
@@ -15,6 +15,7 @@
*/
package com.dremio.plugins.s3.store;
+import static com.dremio.service.users.SystemUser.SYSTEM_USERNAME;
import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY;
import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD;
import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS;
@@ -49,7 +50,6 @@
import com.dremio.exec.store.SchemaConfig;
import com.dremio.exec.store.dfs.FileSystemPlugin;
import com.dremio.exec.store.dfs.FileSystemWrapper;
-import com.dremio.exec.util.ImpersonationUtil;
import com.dremio.plugins.s3.store.copy.S3Constants;
import com.dremio.plugins.util.ContainerFileSystem.ContainerFailure;
import com.dremio.service.namespace.NamespaceKey;
@@ -76,7 +76,7 @@ public class S3StoragePlugin extends FileSystemPlugin {
public static final String NONE_PROVIDER = "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider";
public S3StoragePlugin(S3PluginConfig config, SabotContext context, String name, Provider idProvider) {
- super(config, context, name, null, idProvider);
+ super(config, context, name, idProvider);
}
@Override
@@ -137,7 +137,7 @@ protected List getProperties() {
public SourceState getState() {
try {
ensureDefaultName();
- S3FileSystem fs = getFS(ImpersonationUtil.getProcessUserName()).unwrap(S3FileSystem.class);
+ S3FileSystem fs = getSystemUserFS().unwrap(S3FileSystem.class);
fs.refreshFileSystems();
List failures = fs.getSubFailures();
if(failures.isEmpty()) {
@@ -161,7 +161,8 @@ public SourceState getState() {
private void ensureDefaultName() throws IOException {
String urlSafeName = URLEncoder.encode(getName(), "UTF-8");
getFsConf().set(FileSystem.FS_DEFAULT_NAME_KEY, "dremioS3://" + urlSafeName);
- final FileSystemWrapper fs = getFS(ImpersonationUtil.getProcessUserName());
+ // we create a new fs wrapper since we are calling initialize on it
+ final FileSystemWrapper fs = createFS(SYSTEM_USERNAME);
// do not use fs.getURI() or fs.getConf() directly as they will produce wrong results
fs.initialize(URI.create(getFsConf().get(FileSystem.FS_DEFAULT_NAME_KEY)), getFsConf());
}
@@ -191,7 +192,7 @@ public CreateTableEntry createNewTable(
final CreateTableEntry entry = super.createNewTable(config, key, writerOptions, storageOptions);
- final S3FileSystem fs = getFS(ImpersonationUtil.getProcessUserName()).unwrap(S3FileSystem.class);
+ final S3FileSystem fs = getSystemUserFS().unwrap(S3FileSystem.class);
if (!fs.containerExists(containerName)) {
throw UserException.validationError()
diff --git a/pom.xml b/pom.xml
index 86900f595f..df103e133e 100755
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
pom
Dremio - Root
diff --git a/protocol/pom.xml b/protocol/pom.xml
index b745fe007f..fa99fff33f 100644
--- a/protocol/pom.xml
+++ b/protocol/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio
diff --git a/provision/common/pom.xml b/provision/common/pom.xml
index 21d52ec2de..a2befbb418 100644
--- a/provision/common/pom.xml
+++ b/provision/common/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-provision-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.provision
diff --git a/provision/pom.xml b/provision/pom.xml
index 368eb5cbbd..0c1ec65eb0 100644
--- a/provision/pom.xml
+++ b/provision/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-provision-parent
diff --git a/provision/shimloader/pom.xml b/provision/shimloader/pom.xml
index 039cfabf5a..9a73b31ce0 100644
--- a/provision/shimloader/pom.xml
+++ b/provision/shimloader/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-provision-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.provision
diff --git a/provision/yarn/pom.xml b/provision/yarn/pom.xml
index 5164b06877..685ef38d29 100644
--- a/provision/yarn/pom.xml
+++ b/provision/yarn/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-provision-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.provision
diff --git a/provision/yarn/twill/pom.xml b/provision/yarn/twill/pom.xml
index 636132255f..91f5f5917b 100644
--- a/provision/yarn/twill/pom.xml
+++ b/provision/yarn/twill/pom.xml
@@ -22,7 +22,7 @@
com.dremio.provision
dremio-yarn-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.provision
diff --git a/provision/yarn/yarntwill/pom.xml b/provision/yarn/yarntwill/pom.xml
index 3d42fc9f13..2379b4a749 100644
--- a/provision/yarn/yarntwill/pom.xml
+++ b/provision/yarn/yarntwill/pom.xml
@@ -22,7 +22,7 @@
com.dremio.provision
dremio-yarn-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.provision
diff --git a/sabot/kernel/pom.xml b/sabot/kernel/pom.xml
index abadabb2c4..0a6b349125 100644
--- a/sabot/kernel/pom.xml
+++ b/sabot/kernel/pom.xml
@@ -21,7 +21,7 @@
com.dremio.sabot
dremio-sabot-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-sabot-kernel
Sabot - Kernel
@@ -218,7 +218,7 @@
org.mongodb
mongo-java-driver
- 3.0.2
+ 3.6.4
com.fasterxml.jackson.module
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/ExecConstants.java b/sabot/kernel/src/main/java/com/dremio/exec/ExecConstants.java
index cfe1309216..ec68b0452d 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/ExecConstants.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/ExecConstants.java
@@ -376,4 +376,7 @@ public interface ExecConstants {
LongValidator VOTING_SCHEDULE = new PositiveLongValidator("vote.schedule.millis", Long.MAX_VALUE, 0);
PositiveLongValidator LAST_SEARCH_REINDEX = new PositiveLongValidator("dac.search.last_reindex", Long.MAX_VALUE, 0);
PositiveLongValidator SEARCH_MANAGER_REFRESH_MILLIS = new PositiveLongValidator("dac.search.refresh", Long.MAX_VALUE, TimeUnit.MINUTES.toMillis(1));
+
+ //this option sets the capacity of an ArrowBuf in SlicedBufferManager from which various buffers may be sliced.
+ PowerOfTwoLongValidator BUF_MANAGER_CAPACITY = new PowerOfTwoLongValidator("exec.sliced_bufmgr.capacity", 1 << 24, 1 << 16);
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java b/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java
index 161bd8b34e..ba22d54fb7 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java
@@ -299,7 +299,7 @@ private DremioTable getTableFromPlugin(NamespaceKey key, DatasetConfig datasetCo
final DatasetConfig newDatasetConfig = tableDefinition.getDataset();
final List splits = tableDefinition.getSplits();
NamespaceUtils.copyFromOldConfig(datasetConfig, newDatasetConfig);
- if (BatchSchema.fromDataset(newDatasetConfig).getFieldCount() > maxMetadataColumns) {
+ if (BatchSchema.fromDataset(newDatasetConfig).getTotalFieldCount() > maxMetadataColumns) {
throw UserException.validationError()
.message(String.format("Using datasets with more than %d columns is currently disabled.",
maxMetadataColumns))
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetSaver.java b/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetSaver.java
index b69ae69da2..056837f501 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetSaver.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetSaver.java
@@ -74,7 +74,7 @@ void datasetSave(
try {
DatasetConfig newConfig = accessor.getDataset();
- if (BatchSchema.fromDataset(newConfig).getFieldCount() > maxMetadataLeafColumns) {
+ if (BatchSchema.fromDataset(newConfig).getTotalFieldCount() > maxMetadataLeafColumns) {
NamespaceUtils.copyFromOldConfig(oldConfig, newConfig);
newConfig.setRecordSchema(null);
newConfig.setReadDefinition(null);
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/HLLAccum.java b/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/HLLAccum.java
index d07ce727c3..91f82310ae 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/HLLAccum.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/HLLAccum.java
@@ -20,6 +20,8 @@
import org.apache.arrow.memory.BufferManager;
import org.apache.arrow.vector.holders.ObjectHolder;
+import com.dremio.sabot.exec.context.SlicedBufferManager;
+
import com.yahoo.memory.Memory;
import com.yahoo.memory.WritableMemory;
import com.yahoo.sketches.hll.HllSketch;
@@ -33,12 +35,11 @@
@SuppressWarnings("deprecation")
public final class HLLAccum {
- private final ArrowBuf buf;
private final HllSketch sketch;
private HLLAccum(BufferManager manager, int lgConfigK) {
final int size = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, TgtHllType.HLL_8);
- this.buf = manager.getManagedBuffer(size);
+ final ArrowBuf buf = ((SlicedBufferManager) manager).getManagedBufferSliced(size);
buf.setZero(0, size);
this.sketch = new HllSketch(lgConfigK, TgtHllType.HLL_8, WritableMemory.wrap(buf.nioBuffer(0, size)));
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/UnionAccum.java b/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/UnionAccum.java
index 6ac53ee033..6310db3e6d 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/UnionAccum.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/UnionAccum.java
@@ -18,6 +18,7 @@
import org.apache.arrow.memory.BufferManager;
import org.apache.arrow.vector.holders.ObjectHolder;
+import com.dremio.sabot.exec.context.SlicedBufferManager;
import com.yahoo.memory.Memory;
import com.yahoo.memory.WritableMemory;
import com.yahoo.sketches.hll.HllSketch;
@@ -32,12 +33,11 @@
*/
public class UnionAccum {
- private final ArrowBuf buf;
private final Union union;
private UnionAccum(BufferManager manager, int lgConfigK) {
final int size = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, TgtHllType.HLL_8);
- this.buf = manager.getManagedBuffer(size);
+ final ArrowBuf buf = ((SlicedBufferManager) manager).getManagedBufferSliced(size);
buf.setZero(0, size);
this.union = new Union(lgConfigK, WritableMemory.wrap(buf.nioBuffer(0, size)));
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemDatasetAccessor.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemDatasetAccessor.java
index 77f6fa661e..4c859eb086 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemDatasetAccessor.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemDatasetAccessor.java
@@ -167,7 +167,7 @@ protected final DatasetConfig getDatasetInternal(
if (oldConfig != null && DatasetHelper.getSchemaBytes(oldConfig) != null) {
schema = BatchSchema.fromDataset(oldConfig).merge(newSchema);
}
- if (schema.getFieldCount() > maxLeafColumns) {
+ if (schema.getTotalFieldCount() > maxLeafColumns) {
throw new ColumnCountTooLargeException(
String.format("Using datasets with more than %d columns is currently disabled.", maxLeafColumns));
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemPlugin.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemPlugin.java
index e3cf589d84..f880ca5aa4 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemPlugin.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemPlugin.java
@@ -133,7 +133,7 @@ public class FileSystemPlugin> implements Storage
private final Path basePath;
private final Provider idProvider;
- private FileSystemWrapper fs;
+ private FileSystemWrapper systemUserFS;
private Configuration fsConf;
private FormatPluginOptionExtractor optionExtractor;
protected FormatCreator formatCreator;
@@ -141,11 +141,10 @@ public class FileSystemPlugin> implements Storage
private List dropFileMatchers;
private CompressionCodecFactory codecFactory;
- public FileSystemPlugin(final C config, final SabotContext context, final String name, FileSystemWrapper fs, Provider idProvider) {
+ public FileSystemPlugin(final C config, final SabotContext context, final String name, Provider idProvider) {
this.name = name;
this.config = config;
this.idProvider = idProvider;
- this.fs = fs;
this.context = context;
this.fsConf = getNewFsConf();
this.lpPersistance = context.getLpPersistence();
@@ -156,6 +155,10 @@ public C getConfig(){
return config;
}
+ public boolean supportsColocatedReads() {
+ return true;
+ }
+
@Override
public Iterable getDatasets(String user, DatasetRetrievalOptions ignored) throws Exception {
return Collections.emptyList(); // file system does not know about physical datasets
@@ -163,7 +166,7 @@ public Iterable getDatasets(String user, DatasetRetrieval
@Override
public SourceCapabilities getSourceCapabilities() {
- return fs.isPdfs() ? new SourceCapabilities(REQUIRES_HARD_AFFINITY) : SourceCapabilities.NONE;
+ return systemUserFS.isPdfs() ? new SourceCapabilities(REQUIRES_HARD_AFFINITY) : SourceCapabilities.NONE;
}
public static final Configuration getNewFsConf() {
@@ -178,13 +181,19 @@ public StoragePluginId getId() {
return idProvider.get();
}
- public FileSystemWrapper getFS(String userName) {
- return getFs(userName, null);
+ /**
+ * Create a new {@link FileSystemWrapper} for a given user.
+ *
+ * @param userName
+ * @return
+ */
+ public FileSystemWrapper createFS(String userName) {
+ return createFS(userName, null);
}
- public FileSystemWrapper getFs(String userName, OperatorStats stats) {
+ public FileSystemWrapper createFS(String userName, OperatorStats stats) {
return ImpersonationUtil.createFileSystem(getUGIForUser(userName), getFsConf(), stats,
- getConnectionUniqueProperties());
+ getConnectionUniqueProperties());
}
public UserGroupInformation getUGIForUser(String userName) {
@@ -203,7 +212,7 @@ public Iterable getSubPartitions(List table,
List fileStatuses;
try {
Path fullPath = PathUtils.toFSPath(resolveTableNameToValidPath(table));
- fileStatuses = getFS(schemaConfig.getUserName()).list(fullPath, false);
+ fileStatuses = createFS(schemaConfig.getUserName()).list(fullPath, false);
} catch (IOException e) {
throw new PartitionNotFoundException("Error finding partitions for table " + table, e);
}
@@ -217,10 +226,9 @@ public Class extends StoragePluginRulesFactory> getRulesFactoryClass() {
@Override
public SourceState getState() {
- final FileSystemWrapper fs = getFS(ImpersonationUtil.getProcessUserName());
- if (!fs.isPdfs()) {
+ if (!systemUserFS.isPdfs()) {
try {
- fs.listStatus(config.getPath());
+ systemUserFS.listStatus(config.getPath());
return SourceState.GOOD;
} catch (Exception e) {
return SourceState.badState(e);
@@ -235,7 +243,7 @@ public ViewTable getView(List tableSchemaPath, SchemaConfig schemaConfig
List files = Collections.emptyList();
try {
try {
- files = DotFileUtil.getDotFiles(getFS(schemaConfig.getUserName()), config.getPath(), tableSchemaPath.get(tableSchemaPath.size() - 1), DotFileType.VIEW);
+ files = DotFileUtil.getDotFiles(createFS(schemaConfig.getUserName()), config.getPath(), tableSchemaPath.get(tableSchemaPath.size() - 1), DotFileType.VIEW);
} catch (AccessControlException e) {
if (!schemaConfig.getIgnoreAuthErrors()) {
logger.debug(e.getMessage());
@@ -347,7 +355,7 @@ protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, D
// TODO: why do we need distinguish between system user and process user?
final String userName = config.isImpersonationEnabled() ? SystemUser.SYSTEM_USERNAME : ImpersonationUtil.getProcessUserName();
List parentSchemaPath = new ArrayList<>(fullPath.subList(0, fullPath.size() - 1));
- FileSystemWrapper fs = getFS((user != null) ? user : userName);
+ FileSystemWrapper fs = createFS((user != null) ? user : userName);
FileSelection fileSelection = FileSelection.create(fs, fullPath);
String tableName = datasetPath.getName();
@@ -459,11 +467,8 @@ public void start() throws IOException {
}
// boolean footerNoSeek = contetMutext.getOptionManager().getOption(ExecConstants.PARQUET_FOOTER_NOSEEK);
-
// NOTE: Add fallback format matcher if given in the configuration. Make sure fileMatchers is an order-preserving list.
- if(fs == null) {
- this.fs = getFS(SYSTEM_USERNAME);
- }
+ this.systemUserFS = createFS(SYSTEM_USERNAME);
dropFileMatchers = matchers.subList(0, matchers.size());
createIfNecessary();
@@ -484,10 +489,10 @@ private void createIfNecessary() throws IOException {
try {
// no need to exists here as FileSystemWrapper does an exists check and this is a noop if already existing.
- fs.mkdirs(config.getPath());
+ systemUserFS.mkdirs(config.getPath());
} catch (IOException ex) {
try {
- if(fs.exists(config.getPath())) {
+ if(systemUserFS.exists(config.getPath())) {
// race creation, ignore.
return;
}
@@ -524,7 +529,7 @@ protected FileSystemCachedEntity fromFileStatus(FileStatus status){
public boolean hasAccessPermission(String user, NamespaceKey key, DatasetConfig datasetConfig) {
if (config.isImpersonationEnabled()) {
if (datasetConfig.getReadDefinition() != null) { // allow accessing partial datasets
- final FileSystemWrapper userFs = getFS(user);
+ final FileSystemWrapper userFs = createFS(user);
final List> permissionCheckTasks = Lists.newArrayList();
permissionCheckTasks.addAll(getUpdateKeyPermissionTasks(datasetConfig, userFs));
@@ -642,8 +647,13 @@ protected Boolean runInner() throws Exception {
}
}
- public FileSystemWrapper getFs() {
- return fs;
+ /**
+ * Returns a shared {@link FileSystemWrapper} for the system user.
+ *
+ * @return
+ */
+ public FileSystemWrapper getSystemUserFS() {
+ return systemUserFS;
}
public String getName() {
@@ -654,7 +664,7 @@ public String getName() {
public boolean containerExists(NamespaceKey key) {
final List folderPath = key.getPathComponents();
try {
- return getFS(SYSTEM_USERNAME).isDirectory(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath)));
+ return systemUserFS.isDirectory(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath)));
} catch (IOException e) {
logger.debug("Failure reading path.", e);
return false;
@@ -665,7 +675,7 @@ public boolean containerExists(NamespaceKey key) {
public boolean datasetExists(NamespaceKey key) {
final List filePath = key.getPathComponents();
try {
- return getFS(SYSTEM_USERNAME).exists(PathUtils.toFSPath(resolveTableNameToValidPath(filePath)));
+ return systemUserFS.exists(PathUtils.toFSPath(resolveTableNameToValidPath(filePath)));
} catch (IOException e) {
logger.debug("Failure reading path.", e);
return false;
@@ -673,7 +683,7 @@ public boolean datasetExists(NamespaceKey key) {
}
protected boolean fileExists(String username, List filePath) throws IOException {
- return getFS(username).isFile(PathUtils.toFSPath(resolveTableNameToValidPath(filePath)));
+ return createFS(username).isFile(PathUtils.toFSPath(resolveTableNameToValidPath(filePath)));
}
@Override
@@ -746,7 +756,7 @@ private UpdateStatus checkMultifileStatus(FileUpdateKey fileUpdateKey) {
final Path cachedEntityPath = new Path(cachedEntity.getPath());
try {
- final Optional optionalStatus = fs.getFileStatusSafe(cachedEntityPath);
+ final Optional optionalStatus = systemUserFS.getFileStatusSafe(cachedEntityPath);
if(!optionalStatus.isPresent()) {
// if first entity (root) is missing then table is deleted
if (i == 0) {
@@ -796,7 +806,7 @@ public boolean createOrUpdateView(NamespaceKey key, View view, SchemaConfig sche
}
Path viewPath = getViewPath(key.getPathComponents());
- FileSystemWrapper fs = getFS(schemaConfig.getUserName());
+ FileSystemWrapper fs = createFS(schemaConfig.getUserName());
boolean replaced = fs.exists(viewPath);
final FsPermission viewPerms =
new FsPermission(schemaConfig.getOption(ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY).getStringVal());
@@ -814,7 +824,7 @@ public void dropView(SchemaConfig schemaConfig, List tableSchemaPath) th
.build(logger);
}
- getFS(schemaConfig.getUserName()).delete(getViewPath(tableSchemaPath), false);
+ createFS(schemaConfig.getUserName()).delete(getViewPath(tableSchemaPath), false);
}
private Path getViewPath(List tableSchemaPath) {
@@ -878,7 +888,7 @@ public void dropTable(List tableSchemaPath, SchemaConfig schemaConfig) {
.build(logger);
}
- FileSystemWrapper fs = getFS(schemaConfig.getUserName());
+ FileSystemWrapper fs = createFS(schemaConfig.getUserName());
List fullPath = resolveTableNameToValidPath(tableSchemaPath);
FileSelection fileSelection;
try {
@@ -928,7 +938,7 @@ public void dropTable(List tableSchemaPath, SchemaConfig schemaConfig) {
*/
public SchemaEntity get(List path, String userName) {
try {
- final FileStatus status = getFS(userName).getFileStatus(PathUtils.toFSPath(resolveTableNameToValidPath(path)));
+ final FileStatus status = createFS(userName).getFileStatus(PathUtils.toFSPath(resolveTableNameToValidPath(path)));
final Set> tableNames = Sets.newHashSet();
final NamespaceService ns = context.getNamespaceService(userName);
@@ -951,7 +961,7 @@ public SchemaEntity get(List path, String userName) {
public List list(List folderPath, String userName) {
try {
- final List files = Lists.newArrayList(getFS(userName).listStatus(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath))));
+ final List files = Lists.newArrayList(createFS(userName).listStatus(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath))));
final Set> tableNames = Sets.newHashSet();
final NamespaceService ns = context.getNamespaceService(userName);
final NamespaceKey folderNSKey = new NamespaceKey(folderPath);
@@ -1011,10 +1021,6 @@ public FormatPluginConfig createConfigForTable(String tableName, Map getConnectionUniqueProperties() {
return config.getConnectionUniqueProperties();
}
@@ -1057,7 +1063,7 @@ public CreateTableEntry createNewTable(SchemaConfig config, NamespaceKey key, Wr
// check that there is no directory at the described path.
Path path = resolveTablePathToValidPath(tableName);
try {
- if(fs.exists(path)) {
+ if(systemUserFS.exists(path)) {
throw UserException.validationError().message("Folder already exists at path: %s.", key).build(logger);
}
} catch (IOException e) {
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSConf.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSConf.java
index fd75da16a9..abab8f19e7 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSConf.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSConf.java
@@ -118,7 +118,7 @@ public String getShortCircuitSocketPath() {
@Override
public HDFSStoragePlugin newPlugin(SabotContext context, String name, Provider idProvider) {
- return new HDFSStoragePlugin(this, context, name, null, idProvider);
+ return new HDFSStoragePlugin(this, context, name, idProvider);
}
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSStoragePlugin.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSStoragePlugin.java
index 7c93c5e1f7..d148759f19 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSStoragePlugin.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSStoragePlugin.java
@@ -38,9 +38,8 @@ public class HDFSStoragePlugin extends FileSystemPlugin {
private static final String HDFS_READ_SHORTCIRCUIT_KEY = "dfs.client.read.shortcircuit";
private static final String HDFS_DFS_DOMAIN_SOCKET_PATH_KEY = "dfs.domain.socket.path";
- public HDFSStoragePlugin(HDFSConf config, SabotContext context, String name, FileSystemWrapper fs,
- Provider idProvider) {
- super(config, context, name, fs, idProvider);
+ public HDFSStoragePlugin(HDFSConf config, SabotContext context, String name, Provider idProvider) {
+ super(config, context, name, idProvider);
// TODO Auto-generated constructor stub
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/InternalFileConf.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/InternalFileConf.java
index 3a71011d1b..0730625e32 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/InternalFileConf.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/InternalFileConf.java
@@ -84,7 +84,7 @@ public SchemaMutability getSchemaMutability() {
@Override
public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) {
- return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider);
+ return new FileSystemPlugin<>(this, context, name, pluginIdProvider);
}
public InternalFileConf() {
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/MapRConf.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/MapRConf.java
index ea570e5d1b..83fa97437d 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/MapRConf.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/MapRConf.java
@@ -96,7 +96,7 @@ public SchemaMutability getSchemaMutability() {
@Override
public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) {
- return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider);
+ return new FileSystemPlugin<>(this, context, name, pluginIdProvider);
}
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/NASConf.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/NASConf.java
index c4072c101c..362fa173ae 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/NASConf.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/NASConf.java
@@ -74,7 +74,7 @@ public SchemaMutability getSchemaMutability() {
@Override
public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) {
- return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider);
+ return new FileSystemPlugin<>(this, context, name, pluginIdProvider);
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyScanOperatorCreator.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyScanOperatorCreator.java
index f247e86729..b6c6e348cd 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyScanOperatorCreator.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyScanOperatorCreator.java
@@ -89,7 +89,7 @@ public EasyDatasetSplitXAttr getExtended() {
public ProducerOperator create(FragmentExecutionContext fragmentExecContext, final OperatorContext context, EasySubScan config) throws ExecutionSetupException {
final FileSystemPlugin plugin = fragmentExecContext.getStoragePlugin(config.getPluginId());
- final FileSystemWrapper fs = plugin.getFs(config.getUserName(), context.getStats());
+ final FileSystemWrapper fs = plugin.createFS(config.getUserName(), context.getStats());
final FormatPluginConfig formatConfig = PhysicalDatasetUtils.toFormatPlugin(config.getFileConfig(), Collections.emptyList());
final EasyFormatPlugin> formatPlugin = (EasyFormatPlugin>) plugin.getFormatPlugin(formatConfig);
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyWriter.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyWriter.java
index 815b7f5fbf..700e511d2f 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyWriter.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyWriter.java
@@ -112,6 +112,6 @@ public int getOperatorType() {
@Override
@JsonIgnore
public boolean isPdfs() {
- return plugin.getFs().isPdfs();
+ return plugin.getSystemUserFS().isPdfs();
}
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/ExcelUtil.java b/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/ExcelUtil.java
index b0f518784b..e297b3bc7d 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/ExcelUtil.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/ExcelUtil.java
@@ -35,6 +35,8 @@ public class ExcelUtil {
public static final String ROW = "row";
public static final String CELL = "c";
public static final String VALUE = "v";
+ public static final String INLINE_STRING = "is";
+ public static final String INLINE_STRING_TEXT = "t";
public static final String SST_STRING = "s";
public static final String MERGE_CELLS = "mergeCells";
public static final String MERGE_CELL = "mergeCell";
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/StAXBasedParser.java b/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/StAXBasedParser.java
index f587c7e6ee..a366a9a194 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/StAXBasedParser.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/StAXBasedParser.java
@@ -97,6 +97,7 @@ public int compare(MergeCellRegion o1, MergeCellRegion o2) {
private MinorType valueTypeFromStyle;
private Map styleToTypeCache = Maps.newHashMap();
+ private Map indexToLastTypeCache = Maps.newHashMap();
private final OPCPackage pkgInputStream;
@@ -202,19 +203,23 @@ private void parseHeaders() throws XMLStreamException {
boolean inValue = false;
boolean lookupValueInSST = false;
+ boolean inInlineString = false;
while (xmlStreamReader.hasNext()) {
xmlStreamReader.next();
final int event = xmlStreamReader.getEventType();
switch (event) {
case START_ELEMENT: {
final String name = xmlStreamReader.getLocalName();
- if (ExcelUtil.VALUE.equals(name)) {
+ if (ExcelUtil.VALUE.equals(name) ||
+ (inInlineString && ExcelUtil.INLINE_STRING_TEXT.equals(name))) {
inValue = true;
} else if (ExcelUtil.CELL.equals(name)) {
lookupValueInSST = ExcelUtil.SST_STRING.equals(xmlStreamReader.getAttributeValue(/*namespaceURI=*/null, ExcelUtil.TYPE));
String cellRef = xmlStreamReader.getAttributeValue(/*namespaceURI=*/null, ExcelUtil.CELL_REF);
String columnNameFromRef = ExcelUtil.getColumnName(cellRef);
columnIndex = CellReference.convertColStringToIndex(columnNameFromRef);
+ } else if (ExcelUtil.INLINE_STRING.equals(name)) {
+ inInlineString = true;
}
break;
}
@@ -229,9 +234,12 @@ private void parseHeaders() throws XMLStreamException {
case END_ELEMENT: {
final String name = xmlStreamReader.getLocalName();
- if (ExcelUtil.VALUE.equals(name)) {
+ if (ExcelUtil.VALUE.equals(name) ||
+ (inInlineString && ExcelUtil.INLINE_STRING_TEXT.equals(name))) {
inValue = false;
lookupValueInSST = false;
+ } else if (ExcelUtil.INLINE_STRING.equals(name)) {
+ inInlineString = false;
} else if (ExcelUtil.ROW.equals(name)) {
return;
}
@@ -292,16 +300,20 @@ public State parseNextRecord() throws XMLStreamException {
writer.start();
try {
boolean inValue = false;
+ boolean inInlineString = false;
while (xmlStreamReader.hasNext()) {
xmlStreamReader.next();
final int event = xmlStreamReader.getEventType();
switch (event) {
case START_ELEMENT: {
final String name = xmlStreamReader.getLocalName();
- if (ExcelUtil.VALUE.equals(name)) {
+ if (ExcelUtil.VALUE.equals(name) ||
+ (inInlineString && ExcelUtil.INLINE_STRING_TEXT.equals(name))) {
inValue = true;
} else if (ExcelUtil.CELL.equals(name)) {
handleCellStart();
+ } else if (ExcelUtil.INLINE_STRING.equals(name)) {
+ inInlineString = true;
}
break;
}
@@ -315,7 +327,17 @@ public State parseNextRecord() throws XMLStreamException {
case END_ELEMENT: {
final String name = xmlStreamReader.getLocalName();
- if (ExcelUtil.VALUE.equals(name)) {
+ if (ExcelUtil.VALUE.equals(name) ||
+ (inInlineString && ExcelUtil.INLINE_STRING_TEXT.equals(name))) {
+ currentColumnIndex = -1;
+ lookupNextValueInSST = false;
+ valueTypeFromAttribute = null;
+ valueTypeFromStyle = null;
+ inValue = false;
+ } else if (ExcelUtil.INLINE_STRING.equals(name)) {
+ inInlineString = false;
+ } else if (-1 != currentColumnIndex && ExcelUtil.CELL.equals(name)) {
+ ensureNull();
currentColumnIndex = -1;
lookupNextValueInSST = false;
valueTypeFromAttribute = null;
@@ -405,16 +427,17 @@ private void handleValue(String value) {
final String finalColumnName = columnNameHandler.getColumnName(currentColumnIndex);
- final boolean isProjected = (columnsToProject == null) || (columnsToProject.contains(finalColumnName));
+ final boolean isProjected = (columnsToProject == null) || columnsToProject.contains(finalColumnName);
final MergeCellRegion mergeCellRegion = mergeCells != null ? mergeCells.get(currentCellRef) : null;
switch (valueTypeFromAttribute) {
case BIT:
final int toWrite = "1".equalsIgnoreCase(value) ? 1 : 0;
+ indexToLastTypeCache.put(currentColumnIndex, valueTypeFromAttribute);
if(isProjected) {
- writer.bit(finalColumnName).writeBit(toWrite);
- }
+ writer.bit(finalColumnName).writeBit(toWrite);
+ }
if (mergeCellRegion != null) {
mergeCellRegion.setValue(MinorType.BIT, toWrite);
}
@@ -423,6 +446,7 @@ private void handleValue(String value) {
case FLOAT8:
double dValue = Double.valueOf(value);
if (valueTypeFromStyle == MinorType.TIMESTAMP && DateUtil.isValidExcelDate(dValue)) {
+ indexToLastTypeCache.put(currentColumnIndex, MinorType.TIMESTAMP);
final long dateMillis = DateUtil.getJavaDate(dValue, false, LocaleUtil.TIMEZONE_UTC).getTime();
if(isProjected){
writer.timeStampMilli(finalColumnName).writeTimeStampMilli(dateMillis);
@@ -431,6 +455,7 @@ private void handleValue(String value) {
mergeCellRegion.setValue(MinorType.TIMESTAMP, dateMillis);
}
} else {
+ indexToLastTypeCache.put(currentColumnIndex, valueTypeFromAttribute);
if(isProjected) {
writer.float8(finalColumnName).writeFloat8(dValue);
}
@@ -441,6 +466,7 @@ private void handleValue(String value) {
break;
case VARCHAR:
+ indexToLastTypeCache.put(currentColumnIndex, valueTypeFromAttribute);
final byte[] b = value.getBytes(Charsets.UTF_8);
managedBuf = managedBuf.reallocIfNeeded(b.length);
managedBuf.setBytes(0, b);
@@ -454,8 +480,51 @@ private void handleValue(String value) {
default:
throw UserException.unsupportedError()
- .message("Unknown type received %s", valueTypeFromAttribute)
- .build(logger);
+ .message("Unknown type received %s", valueTypeFromAttribute)
+ .build(logger);
+ }
+ }
+ }
+
+ /**
+ * Ensure a null value is set for the writer by retrieving the writer for the column but not writing a value.
+ */
+ private void ensureNull() {
+ assert currentColumnIndex != -1 : "Invalid currentColumnIndex";
+
+ final String finalColumnName = columnNameHandler.getColumnName(currentColumnIndex);
+
+ if ((columnsToProject == null) || columnsToProject.contains(finalColumnName)) {
+ MinorType type = indexToLastTypeCache.get(currentColumnIndex);
+ if (type == null) {
+ if (valueTypeFromAttribute == MinorType.FLOAT8 && valueTypeFromStyle == MinorType.TIMESTAMP) {
+ type = MinorType.TIMESTAMP;
+ } else {
+ type = valueTypeFromAttribute;
+ }
+ }
+
+ switch (type) {
+ case BIT:
+ writer.bit(finalColumnName);
+ break;
+
+ case FLOAT8:
+ writer.float8(finalColumnName);
+ break;
+
+ case TIMESTAMP:
+ writer.timeMilli(finalColumnName);
+ break;
+
+ case VARCHAR:
+ writer.varChar(finalColumnName);
+ break;
+
+ default:
+ throw UserException.unsupportedError()
+ .message("Unknown type received %s", valueTypeFromAttribute)
+ .build(logger);
}
}
}
@@ -482,20 +551,22 @@ private String resolveValue(String value, boolean lookupNextValueInSST) {
* current row onwards.
*/
private void handleMergedCells() {
- Iterator iter = mergeCellsSortedByRowEnd.iterator();
- while(iter.hasNext()) {
- final MergeCellRegion mcr = iter.next();
- if (mcr.hasValue() && mcr.containsRow(runningRecordCount)) {
- for(int colIndex = mcr.colStart; colIndex <= mcr.colEnd; colIndex++) {
- String colName = columnNameHandler.getColumnName(colIndex);
- if(columnsToProject == null || columnsToProject.contains(colName)) {
- mcr.write(managedBuf, writer, colName);
+ if (mergeCellsSortedByRowEnd != null) {
+ Iterator iter = mergeCellsSortedByRowEnd.iterator();
+ while (iter.hasNext()) {
+ final MergeCellRegion mcr = iter.next();
+ if (mcr.hasValue() && mcr.containsRow(runningRecordCount)) {
+ for (int colIndex = mcr.colStart; colIndex <= mcr.colEnd; colIndex++) {
+ String colName = columnNameHandler.getColumnName(colIndex);
+ if (columnsToProject == null || columnsToProject.contains(colName)) {
+ mcr.write(managedBuf, writer, colName);
+ }
}
}
- }
- if (mcr.rowEnd < runningRecordCount) {
- iter.remove();
+ if (mcr.rowEnd < runningRecordCount) {
+ iter.remove();
+ }
}
}
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetFormatDatasetAccessor.java b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetFormatDatasetAccessor.java
index 099e88526b..a73ca8d737 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetFormatDatasetAccessor.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetFormatDatasetAccessor.java
@@ -175,7 +175,7 @@ public BatchSchema getBatchSchema(final FileSelection selection, final FileSyste
mutator.getContainer().buildSchema(BatchSchema.SelectionVectorMode.NONE);
final BatchSchema schema = mutator.getContainer().getSchema();
- if (schema.getFieldCount() > maxLeafColumns) {
+ if (schema.getTotalFieldCount() > maxLeafColumns) {
throw new ColumnCountTooLargeException(
String.format("Using datasets with more than %d columns is currently disabled.", maxLeafColumns));
}
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetOperatorCreator.java b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetOperatorCreator.java
index 42ec19d5e8..b1ba248fad 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetOperatorCreator.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetOperatorCreator.java
@@ -60,7 +60,7 @@ public class ParquetOperatorCreator implements Creator {
@Override
public ProducerOperator create(FragmentExecutionContext fragmentExecContext, final OperatorContext context, final ParquetSubScan config) throws ExecutionSetupException {
final FileSystemPlugin plugin = fragmentExecContext.getStoragePlugin(config.getPluginId());
- final FileSystemWrapper fs = plugin.getFs(config.getUserName(), context.getStats());
+ final FileSystemWrapper fs = plugin.createFS(config.getUserName(), context.getStats());
final Stopwatch watch = Stopwatch.createStarted();
diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetWriter.java b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetWriter.java
index 1c7879a63c..2255cea18e 100644
--- a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetWriter.java
+++ b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetWriter.java
@@ -108,7 +108,7 @@ public int getOperatorType() {
@Override
@JsonIgnore
public boolean isPdfs() {
- return plugin.getFs().isPdfs();
+ return plugin.getSystemUserFS().isPdfs();
}
@JsonIgnore
diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/BufferManagerImpl.java b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/BufferManagerImpl.java
index 924271dbeb..dde05da409 100644
--- a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/BufferManagerImpl.java
+++ b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/BufferManagerImpl.java
@@ -17,21 +17,41 @@
import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.BufferManager;
import com.carrotsearch.hppc.LongObjectHashMap;
import com.carrotsearch.hppc.predicates.LongObjectPredicate;
import io.netty.buffer.ArrowBuf;
-public class BufferManagerImpl implements BufferManager {
+public class BufferManagerImpl implements SlicedBufferManager {
private LongObjectHashMap managedBuffers = new LongObjectHashMap<>();
private final BufferAllocator allocator;
+ /**
+ * This class keeps a pre-allocated large buffer & serve any subsequent allocation
+ * requests via slicing from this large buffer.
+ * This helps in reducing the heap allocation overhead as sliced ArrowBuf has smaller footprint.
+ * Also, the allocation of large buffer may be optimized by doing power-of-2 allocations.
+ *
+ * largeBufCapacity: the capacity of the largeBuf
+ * largeBufUsed tracks: how much memory has been sliced away from the largeBuf
+ */
+ private int largeBufCapacity;
+ private int largeBufUsed;
+ private ArrowBuf largeBuf;
+
+
public BufferManagerImpl(BufferAllocator allocator) {
this.allocator = allocator;
}
+ public BufferManagerImpl(BufferAllocator allocator, int largeBufCapacity) {
+ this.allocator = allocator;
+ this.largeBufCapacity = largeBufCapacity;
+ this.largeBuf = null;
+ this.largeBufUsed = 0;
+ }
+
@Override
public void close() {
managedBuffers.forEach(new LongObjectPredicate() {
@@ -61,4 +81,23 @@ public ArrowBuf getManagedBuffer(int size) {
managedBuffers.put(newBuf.memoryAddress(), newBuf);
return newBuf;
}
+
+ public ArrowBuf getManagedBufferSliced(int size) {
+
+ if (size >= largeBufCapacity) {
+ return getManagedBuffer(size);
+ }
+
+ final int availableSpace = (largeBufCapacity - largeBufUsed);
+
+ if (size > availableSpace || largeBuf == null) {
+ largeBuf = allocator.buffer(largeBufCapacity, this);
+ managedBuffers.put(largeBuf.memoryAddress(), largeBuf);
+ largeBufUsed = 0;
+ }
+
+ final ArrowBuf buf = largeBuf.slice(largeBufUsed, size);
+ largeBufUsed += size;
+ return buf;
+ }
}
diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/OperatorContextImpl.java b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/OperatorContextImpl.java
index b64a1215ea..8b80c34897 100644
--- a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/OperatorContextImpl.java
+++ b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/OperatorContextImpl.java
@@ -25,6 +25,7 @@
import com.dremio.common.AutoCloseables;
import com.dremio.common.config.SabotConfig;
+import com.dremio.exec.ExecConstants;
import com.dremio.exec.compile.CodeCompiler;
import com.dremio.exec.expr.ClassProducer;
import com.dremio.exec.expr.ClassProducerImpl;
@@ -94,7 +95,12 @@ public OperatorContextImpl(
this.allocator = allocator;
this.fragmentOutputAllocator = fragmentOutputAllocator;
this.popConfig = popConfig;
- this.manager = new BufferManagerImpl(allocator);
+
+ //some unit test cases pass null optionManager
+ final int bufCapacity = (optionManager != null) ?
+ (int)optionManager.getOption(ExecConstants.BUF_MANAGER_CAPACITY) : (1 << 16);
+ this.manager = new BufferManagerImpl(allocator, bufCapacity);
+
this.stats = stats;
this.executionControls = executionControls;
this.executor = executor;
diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/SlicedBufferManager.java b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/SlicedBufferManager.java
new file mode 100644
index 0000000000..a2a149f1c9
--- /dev/null
+++ b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/SlicedBufferManager.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2017-2018 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dremio.sabot.exec.context;
+
+import org.apache.arrow.memory.BufferManager;
+
+import io.netty.buffer.ArrowBuf;
+
+
+public interface SlicedBufferManager extends BufferManager {
+
+ ArrowBuf getManagedBufferSliced(int size);
+
+}
diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/hash/HashAggOperator.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/hash/HashAggOperator.java
index 6e0339b04e..2c3ea68496 100644
--- a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/hash/HashAggOperator.java
+++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/hash/HashAggOperator.java
@@ -83,6 +83,9 @@ public class HashAggOperator implements SingleInputOperator {
public static final TypeValidators.PositiveLongValidator HASHAGG_MINMAX_CARDINALITY_LIMIT = new TypeValidators.PositiveLongValidator("exec.operator.aggregate.minmax_cardinality_limit", Long.MAX_VALUE, 10000);
+ //this option sets the capacity of an ArrowBuf in BufferManager from which various buffers may be sliced.
+ public static final TypeValidators.PowerOfTwoLongValidator BUF_MANAGER_CAPACITY = new TypeValidators.PowerOfTwoLongValidator("exec.operator.aggregate.bufmgr.capacity", 1 << 24, 1 << 16);
+
private static final ControlsInjector injector = ControlsInjectorFactory.getInjector(HashAggOperator.class);
@VisibleForTesting
diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggDebug.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggDebug.java
index c8f5c23105..b8c9175662 100644
--- a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggDebug.java
+++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggDebug.java
@@ -168,6 +168,8 @@ void setInfoAfterInit(final long totalPreallocatedMemory,
this.aggSchema = (detailedEventTracing ? outgoing.toString() : "enable tracing to record schema");
}
+ void setMaxVarBlockLength(final int maxVarLen) { this.maxVarBlockLength = maxVarLen; }
+
void recordOOMEvent(final int iterations,
final int ooms,
final long currentAllocatedMemory,
diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggOperator.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggOperator.java
index 5a63026338..b7d280aaa1 100644
--- a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggOperator.java
+++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggOperator.java
@@ -472,6 +472,7 @@ private void setLocalInfoForHashTable() {
* the info we need.
*/
this.maxVariableBlockLength = hashAggPartitions[0].hashTable.getVariableBlockMaxLength();
+ debug.setMaxVarBlockLength(this.maxVariableBlockLength); //for debugging purpose
this.bitsInChunk = hashAggPartitions[0].hashTable.getBitsInChunk();
this.chunkOffsetMask = hashAggPartitions[0].hashTable.getChunkOffsetMask();
/* hashtable should not have changed the configured value of VECTORIZED_HASHAGG_MAXBATCHSIZE */
@@ -790,11 +791,11 @@ private void consumeDataHelper(final int records) throws Exception {
recordsPivoted = BoundedPivots.pivot(pivot, recordsConsumed, stepSize, fixedBlockVector, variableBlockVector);
pivotWatch.stop();
+
/* STEP 2: then we hash partition the dataset and add pivoted data to multiple hash tables */
long partitionsUsed = insertIntoPartitions(records, recordsPivoted, keyFixedVectorAddr,
keyVarVectorAddr, recordsConsumed, 0, 0, false, 0);
-
if (internalStateMachine == InternalState.SPILL_NEXT_BATCH) {
/* insertion for this set of pivoted records failed in between.
* we will resume later from the failure point for these set of records and any
diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/common/ht2/BoundedPivots.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/common/ht2/BoundedPivots.java
index aa0bbfd7fd..1407e1d391 100644
--- a/sabot/kernel/src/main/java/com/dremio/sabot/op/common/ht2/BoundedPivots.java
+++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/common/ht2/BoundedPivots.java
@@ -143,6 +143,11 @@ private static int pivotVariableLengths(
outputRecordIdx++;
}
+ if (outputRecordIdx == 0) {
+ throw new StringIndexOutOfBoundsException("Not enough space to pivot single record. Allocated capacity for pivot: "
+ + Long.toString(targetVariable.getMaxMemoryAddress() - targetVariable.getMemoryAddress()) + " bytes.");
+ }
+
return outputRecordIdx;
}
diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/writer/WriterCommitterOperator.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/writer/WriterCommitterOperator.java
index 4ce0181c3f..d1867cc5bd 100644
--- a/sabot/kernel/src/main/java/com/dremio/sabot/op/writer/WriterCommitterOperator.java
+++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/writer/WriterCommitterOperator.java
@@ -70,7 +70,7 @@ public VectorAccessible setup(VectorAccessible accessible) throws Exception {
throw new IllegalStateException(String.format("Incoming record writer schema doesn't match intended. Expected %s, received %s", RecordWriter.SCHEMA, schema));
}
- fs = config.getPlugin().getFS(config.getUserName());
+ fs = config.getPlugin().createFS(config.getUserName());
// replacement expression.
LogicalExpression replacement;
diff --git a/sabot/kernel/src/test/java/com/dremio/exec/store/dfs/TestHDFSStoragePlugin.java b/sabot/kernel/src/test/java/com/dremio/exec/store/dfs/TestHDFSStoragePlugin.java
index c2cf3f4b19..b70c0856d4 100644
--- a/sabot/kernel/src/test/java/com/dremio/exec/store/dfs/TestHDFSStoragePlugin.java
+++ b/sabot/kernel/src/test/java/com/dremio/exec/store/dfs/TestHDFSStoragePlugin.java
@@ -49,7 +49,7 @@ public void testHdfsConfApplied() throws Exception {
when(context.getClasspathScan()).thenReturn(DremioTest.CLASSPATH_SCAN_RESULT);
Provider idProvider = () -> { return new StoragePluginId(null, conf, null); };
- try(HDFSStoragePlugin fileSystemPlugin = new HDFSStoragePlugin(conf, context, "test-plugin", null, idProvider)) {
+ try(HDFSStoragePlugin fileSystemPlugin = new HDFSStoragePlugin(conf, context, "test-plugin", idProvider)) {
fileSystemPlugin.start();
final Configuration fsConf = fileSystemPlugin.getFsConf();
diff --git a/sabot/kernel/src/test/java/com/dremio/exec/store/excel/TestXlsxExcelFormatPlugin.java b/sabot/kernel/src/test/java/com/dremio/exec/store/excel/TestXlsxExcelFormatPlugin.java
index 6a7ecaec1a..5b71d3fe1b 100644
--- a/sabot/kernel/src/test/java/com/dremio/exec/store/excel/TestXlsxExcelFormatPlugin.java
+++ b/sabot/kernel/src/test/java/com/dremio/exec/store/excel/TestXlsxExcelFormatPlugin.java
@@ -54,9 +54,18 @@ public void testEmpty2Xls() throws Exception {
final String filePath = getExcelDir() + "empty2.xlsx";
final String query = String.format("SELECT * FROM TABLE(dfs.\"%s\" (type => 'excel', extractHeader => true, hasMergedCells => true, xls => false))", filePath);
- testAndExpectUserException(query, ErrorType.DATA_READ, "Selected table has no columns.");
+ // There are actual columns in this table, but they contain no value. Hence, show the columns with nulls.
+ test(query);
}
+ @Test
+ public void testInlineString() throws Exception {
+ final String filePath = getExcelDir() + "InlineString.xlsx";
+ final String query = String.format("SELECT * FROM TABLE(dfs.\"%s\" (type => 'excel', extractHeader => true, hasMergedCells => false, xls => false))", filePath);
+
+ // This will fail if the inline string column is ignored, as no columns will be detected.
+ test(query);
+ }
@Test
public void testHeaderOnly() throws Exception {
diff --git a/sabot/kernel/src/test/java/com/dremio/sabot/CustomHashAggDataGenerator.java b/sabot/kernel/src/test/java/com/dremio/sabot/CustomHashAggDataGenerator.java
index 9ff06940d5..2ff5cd556c 100644
--- a/sabot/kernel/src/test/java/com/dremio/sabot/CustomHashAggDataGenerator.java
+++ b/sabot/kernel/src/test/java/com/dremio/sabot/CustomHashAggDataGenerator.java
@@ -48,17 +48,17 @@
public class CustomHashAggDataGenerator implements Generator {
/* fixed width dimensions */
- private final static Field FIXKEY1 = CompleteType.INT.toField("FIXKEY1");
- private final static Field FIXKEY2 = CompleteType.BIGINT.toField("FIXKEY2");
+ private static Field FIXKEY1 = CompleteType.INT.toField("FIXKEY1");
+ private static Field FIXKEY2 = CompleteType.BIGINT.toField("FIXKEY2");
/* variable width dimensions */
- private final static Field VARKEY1 = CompleteType.VARCHAR.toField("VARKEY1");
+ private static Field VARKEY1 = CompleteType.VARCHAR.toField("VARKEY1");
/* Measures */
- private final static Field MEASURE1 = CompleteType.INT.toField("MEASURE1");
- private final static Field MEASURE2 = CompleteType.BIGINT.toField("MEASURE2");
- private final static Field MEASURE3 = CompleteType.FLOAT.toField("MEASURE3");
- private final static Field MEASURE4 = CompleteType.DOUBLE.toField("MEASURE4");
+ private static Field MEASURE1 = CompleteType.INT.toField("MEASURE1");
+ private static Field MEASURE2 = CompleteType.BIGINT.toField("MEASURE2");
+ private static Field MEASURE3 = CompleteType.FLOAT.toField("MEASURE3");
+ private static Field MEASURE4 = CompleteType.DOUBLE.toField("MEASURE4");
/* input table */
private Integer[] fixKeyValues1;
@@ -69,24 +69,24 @@ public class CustomHashAggDataGenerator implements Generator {
private Float[] measure3;
private Double[] measure4;
- private final VectorContainer container;
+ private VectorContainer container;
/* fixed width key columns */
- private final IntVector fixkey1;
- private final BigIntVector fixkey2;
+ private IntVector fixkey1;
+ private BigIntVector fixkey2;
/* variable width key columns */
- private final VarCharVector varkey1;
+ private VarCharVector varkey1;
/* measure columns */
- private final IntVector m1;
- private final BigIntVector m2;
- private final Float4Vector m3;
- private final Float8Vector m4;
+ private IntVector m1;
+ private BigIntVector m2;
+ private Float4Vector m3;
+ private Float8Vector m4;
private int position;
private int batches;
- private final boolean largeVarChars;
+ private boolean largeVarChars;
private static final int GROUP_REPEAT_PER_BATCH = 5;
/* simplify data generation by using a fixed internal batch size */
@@ -104,13 +104,36 @@ public class CustomHashAggDataGenerator implements Generator {
private static final String VARCHAR_BASEVALUE = "Dremio-3.0";
private static final int GROUP_INTERVAL_PER_BATCH = 20;
+ private int numRows;
private final HashMap aggregatedResults = new HashMap<>();
+ private int minLargeVarCharLen = 0;
+
+ private void InternalInit(int numRows, BufferAllocator allocator, final boolean largeVarChars)
+ {
+ this.numRows = numRows;
+ this.batches = numRows/BATCH_SIZE;
+ this.largeVarChars = largeVarChars;
+ createBigSchemaAndInputContainer(allocator);
+ }
+
public CustomHashAggDataGenerator(int numRows, BufferAllocator allocator,
final boolean largeVarChars) {
+ Preconditions.checkState(numRows > 0 && numRows%BATCH_SIZE == 0,
+ "ERROR: total number of rows should be greater than 0");
+ InternalInit(numRows, allocator, largeVarChars);
+ }
+
+ public CustomHashAggDataGenerator(int numRows, BufferAllocator allocator,
+ final int minVarCharLen /*minimum length of long varchar*/) {
Preconditions.checkState(numRows > 0 && numRows%BATCH_SIZE == 0,
"ERROR: total number of rows should be greater than 0");
+ this.minLargeVarCharLen = minVarCharLen;
+ InternalInit(numRows, allocator, true);
+ }
+
+ private void createBigSchemaAndInputContainer(final BufferAllocator allocator) {
BatchSchema schema = BatchSchema.newBuilder()
.addField(FIXKEY1)
.addField(FIXKEY2)
@@ -199,7 +222,11 @@ private void buildInputDataset(int numRows) {
/* key columns */
fixKeyValues1[0] = INT_BASEVALUE;
fixKeyValues2[0] = BIGINT_BASEVALUE;
- varKeyValues1[0] = VARCHAR_BASEVALUE;
+ if (largeVarChars && minLargeVarCharLen != 0) {
+ varKeyValues1[0] = RandomStringUtils.randomAlphabetic(minLargeVarCharLen);
+ } else {
+ varKeyValues1[0] = VARCHAR_BASEVALUE;
+ }
/* measure columns */
measure1[0] = INT_BASEVALUE;
diff --git a/sabot/kernel/src/test/java/com/dremio/sabot/aggregate/hash/TestSpillingHashAgg.java b/sabot/kernel/src/test/java/com/dremio/sabot/aggregate/hash/TestSpillingHashAgg.java
index 5b157b26d8..e958587d19 100644
--- a/sabot/kernel/src/test/java/com/dremio/sabot/aggregate/hash/TestSpillingHashAgg.java
+++ b/sabot/kernel/src/test/java/com/dremio/sabot/aggregate/hash/TestSpillingHashAgg.java
@@ -21,6 +21,7 @@
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
+import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
@@ -82,6 +83,92 @@ public void testNoSpill() throws Exception {
}
}
+ /**
+ * Tests with varchar key of length > 32k
+ * @throws Exception
+ */
+ @Test
+ public void testVeryLargeVarcharKey() throws Exception {
+ final HashAggregate agg = getHashAggregate();
+ agg.setInitialAllocation(1_000_000);
+ agg.setMaxAllocation(12_000_000);
+
+ boolean exceptionThrown = false;
+
+
+ final int shortLen = (30 * 1024);
+ final int largeLen = (32 * 1024);
+
+ //30k size must not fail, any subsequent 32k inserts would fail
+ boolean shortLenSucess = false;
+ try (AutoCloseable useSpillingAgg = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_USE_SPILLING_OPERATOR, true)) {
+ try (CustomHashAggDataGenerator generator = new CustomHashAggDataGenerator(1000, getTestAllocator(), shortLen)) {
+ try (CustomHashAggDataGenerator generator1 = new CustomHashAggDataGenerator(1000, getTestAllocator(), largeLen)) {
+ Fixtures.Table table = generator.getExpectedGroupsAndAggregations();
+ validateSingle(agg, VectorizedHashAggOperator.class, generator, table, 1000);
+
+ //30k key size should have worked. must reach here!
+ Assert.assertEquals(0, 0);
+
+ shortLenSucess = true;
+ //add some 32k keys
+ Fixtures.Table table1 = generator1.getExpectedGroupsAndAggregations();
+ validateSingle(agg, VectorizedHashAggOperator.class, generator1, table1, 1000);
+
+ //must not reach here
+ Assert.assertEquals(0, 1);
+ }
+ }
+ } catch (StringIndexOutOfBoundsException userExp) {
+ exceptionThrown = true;
+ Assert.assertEquals(true, shortLenSucess);
+ } finally {
+ Assert.assertEquals(true, exceptionThrown);
+ }
+
+ //fails to pivot with 32k key size
+ try (AutoCloseable useSpillingAgg = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_USE_SPILLING_OPERATOR, true)) {
+ try (CustomHashAggDataGenerator generator = new CustomHashAggDataGenerator(1000, getTestAllocator(), largeLen)) {
+ Fixtures.Table table = generator.getExpectedGroupsAndAggregations();
+ validateSingle(agg, VectorizedHashAggOperator.class, generator, table, 1000);
+
+ //must not reach here
+ Assert.assertEquals(0, 1);
+ }
+ } catch (StringIndexOutOfBoundsException userExp) {
+ exceptionThrown = true;
+ } finally {
+ Assert.assertEquals(true, exceptionThrown);
+ }
+
+ //passes 32k key size with increased batch of 2k
+ exceptionThrown = false;
+ try (AutoCloseable useSpillingAgg = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_USE_SPILLING_OPERATOR, true)) {
+ try (CustomHashAggDataGenerator generator = new CustomHashAggDataGenerator(1000, getTestAllocator(), largeLen);
+ AutoCloseable options = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_BATCHSIZE, 2048)) {
+ Fixtures.Table table = generator.getExpectedGroupsAndAggregations();
+ validateSingle(agg, VectorizedHashAggOperator.class, generator, table, 1000);
+ }
+ } catch (Exception e) {
+ exceptionThrown = true;
+ } finally {
+ Assert.assertEquals(false, exceptionThrown);
+ }
+
+ //should fail with 1MB key size
+ try (AutoCloseable useSpillingAgg = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_USE_SPILLING_OPERATOR, true)) {
+ try (CustomHashAggDataGenerator generator = new CustomHashAggDataGenerator(1000, getTestAllocator(), (1024 * 1024));
+ AutoCloseable options = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_BATCHSIZE, 4096)) {
+ Fixtures.Table table = generator.getExpectedGroupsAndAggregations();
+ validateSingle(agg, VectorizedHashAggOperator.class, generator, table, 1000);
+ }
+ } catch (StringIndexOutOfBoundsException userExp) {
+ exceptionThrown = true;
+ } finally {
+ Assert.assertEquals(true, exceptionThrown);
+ }
+ }
+
/**
* Test failure during operator setup when provided memory constraints
* are lower than the memory required for preallocating data structures.
diff --git a/sabot/kernel/src/test/resources/excel/InlineString.xlsx b/sabot/kernel/src/test/resources/excel/InlineString.xlsx
new file mode 100644
index 0000000000..cd1683f344
Binary files /dev/null and b/sabot/kernel/src/test/resources/excel/InlineString.xlsx differ
diff --git a/sabot/logical/pom.xml b/sabot/logical/pom.xml
index 6eef9158a9..4cb5e9e8e9 100644
--- a/sabot/logical/pom.xml
+++ b/sabot/logical/pom.xml
@@ -22,7 +22,7 @@
com.dremio.sabot
dremio-sabot-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-sabot-logical
diff --git a/sabot/pom.xml b/sabot/pom.xml
index f9f590d623..cdfa1c7648 100644
--- a/sabot/pom.xml
+++ b/sabot/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.sabot
diff --git a/services/accelerator/pom.xml b/services/accelerator/pom.xml
index 022d2db5e0..e3ba6f4228 100644
--- a/services/accelerator/pom.xml
+++ b/services/accelerator/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-accelerator
diff --git a/services/accelerator/src/main/java/com/dremio/service/reflection/materialization/AccelerationStoragePlugin.java b/services/accelerator/src/main/java/com/dremio/service/reflection/materialization/AccelerationStoragePlugin.java
index 74275117cb..e762cddd75 100644
--- a/services/accelerator/src/main/java/com/dremio/service/reflection/materialization/AccelerationStoragePlugin.java
+++ b/services/accelerator/src/main/java/com/dremio/service/reflection/materialization/AccelerationStoragePlugin.java
@@ -69,7 +69,7 @@ public class AccelerationStoragePlugin extends FileSystemPlugin idProvider) {
- super(config, context, name, null, idProvider);
+ super(config, context, name, idProvider);
}
@Override
@@ -166,7 +166,7 @@ public SourceTableDefinition getDataset(
@Override
public Iterable apply(Refresh input) {
try {
- FileSelection selection = FileSelection.create(getFs(), resolveTablePathToValidPath(input.getPath()));
+ FileSelection selection = FileSelection.create(getSystemUserFS(), resolveTablePathToValidPath(input.getPath()));
if(selection != null) {
return selection.minusDirectories().getFileStatuses();
}
@@ -177,7 +177,7 @@ public Iterable apply(Refresh input) {
}}).toList();
FileSelection selection = FileSelection.createFromExpanded(allStatus, selectionRoot);
- return new ParquetFormatDatasetAccessor(oldConfig, getFs(), selection, this, datasetPath, EMPTY, formatPlugin,
+ return new ParquetFormatDatasetAccessor(oldConfig, getSystemUserFS(), selection, this, datasetPath, EMPTY, formatPlugin,
options.maxMetadataLeafColumns());
}
diff --git a/services/base-rpc/pom.xml b/services/base-rpc/pom.xml
index 3c4715ba7b..5b29f09d6d 100644
--- a/services/base-rpc/pom.xml
+++ b/services/base-rpc/pom.xml
@@ -21,7 +21,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-base-rpc
Services - Base RPC
diff --git a/services/configuration/pom.xml b/services/configuration/pom.xml
index 0416bea45c..8883a55457 100644
--- a/services/configuration/pom.xml
+++ b/services/configuration/pom.xml
@@ -22,7 +22,7 @@
dremio-services-parent
com.dremio.services
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
4.0.0
dremio-services-configuration
diff --git a/services/coordinator/pom.xml b/services/coordinator/pom.xml
index f902b90289..63c05255b8 100644
--- a/services/coordinator/pom.xml
+++ b/services/coordinator/pom.xml
@@ -21,7 +21,7 @@
dremio-services-parent
com.dremio.services
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-coordinator
Services - Coordinator
diff --git a/services/datastore/pom.xml b/services/datastore/pom.xml
index 906809ccf2..39fb53fe5d 100644
--- a/services/datastore/pom.xml
+++ b/services/datastore/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-datastore
Services - Datastore
diff --git a/services/fabric-rpc/pom.xml b/services/fabric-rpc/pom.xml
index 4b96f1d666..2cb24af383 100644
--- a/services/fabric-rpc/pom.xml
+++ b/services/fabric-rpc/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-fabric-rpc
Services - Fabric RPC
diff --git a/services/jobs/pom.xml b/services/jobs/pom.xml
index 98011533ee..86a4143884 100644
--- a/services/jobs/pom.xml
+++ b/services/jobs/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-jobs
Services - Jobs
diff --git a/services/jobs/src/main/java/com/dremio/service/jobs/JobResultsStore.java b/services/jobs/src/main/java/com/dremio/service/jobs/JobResultsStore.java
index e5b821bc46..f020bbf6bb 100644
--- a/services/jobs/src/main/java/com/dremio/service/jobs/JobResultsStore.java
+++ b/services/jobs/src/main/java/com/dremio/service/jobs/JobResultsStore.java
@@ -36,7 +36,6 @@
import com.dremio.datastore.IndexedStore;
import com.dremio.exec.store.dfs.FileSystemPlugin;
import com.dremio.exec.store.easy.arrow.ArrowFileMetadata;
-import com.dremio.exec.util.ImpersonationUtil;
import com.dremio.service.Service;
import com.dremio.service.job.proto.JobAttempt;
import com.dremio.service.job.proto.JobId;
@@ -75,7 +74,7 @@ public class JobResultsStore implements Service {
public JobResultsStore(final FileSystemPlugin plugin, final IndexedStore store,
final BufferAllocator allocator) throws IOException {
this.storageName = plugin.getName();
- this.dfs = plugin.getFS(ImpersonationUtil.getProcessUserName());
+ this.dfs = plugin.getSystemUserFS();
this.jobStoreLocation = plugin.getConfig().getPath();
this.dfs.mkdirs(jobStoreLocation);
this.store = store;
diff --git a/services/metrics/pom.xml b/services/metrics/pom.xml
index 6207f33ea8..b8e78835a2 100644
--- a/services/metrics/pom.xml
+++ b/services/metrics/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-metrics
Services - Metrics
diff --git a/services/namespace/pom.xml b/services/namespace/pom.xml
index 9574eae105..3074249303 100644
--- a/services/namespace/pom.xml
+++ b/services/namespace/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-namespace
Services - Namespace
diff --git a/services/options/pom.xml b/services/options/pom.xml
index 3bdc714084..509c7fa9ea 100644
--- a/services/options/pom.xml
+++ b/services/options/pom.xml
@@ -23,7 +23,7 @@
dremio-services-parent
com.dremio.services
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-options
diff --git a/services/pom.xml b/services/pom.xml
index 2195102f99..5c293630a6 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -22,7 +22,7 @@
com.dremio
dremio-root
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.services
diff --git a/services/resourcescheduler/pom.xml b/services/resourcescheduler/pom.xml
index 025c5ea056..7a6d2c38cc 100644
--- a/services/resourcescheduler/pom.xml
+++ b/services/resourcescheduler/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-resourcescheduler
Services - Resource Scheduler
diff --git a/services/scheduler/pom.xml b/services/scheduler/pom.xml
index 4c28d8af14..1263902979 100644
--- a/services/scheduler/pom.xml
+++ b/services/scheduler/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-scheduler
Services - Task Scheduler
diff --git a/services/spill/pom.xml b/services/spill/pom.xml
index f158c03765..02ccbd2842 100644
--- a/services/spill/pom.xml
+++ b/services/spill/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-spill
Services - Spill management
diff --git a/services/users/pom.xml b/services/users/pom.xml
index d67ad6b8c1..db275f80de 100644
--- a/services/users/pom.xml
+++ b/services/users/pom.xml
@@ -22,7 +22,7 @@
com.dremio.services
dremio-services-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
dremio-services-users
diff --git a/tools/build-tools/pom.xml b/tools/build-tools/pom.xml
index 85860d7bf3..98549305ad 100644
--- a/tools/build-tools/pom.xml
+++ b/tools/build-tools/pom.xml
@@ -23,7 +23,7 @@
com.dremio.tools
dremio-build-tools
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
jar
Tools - Build tools
diff --git a/tools/fmpp-maven-plugin/pom.xml b/tools/fmpp-maven-plugin/pom.xml
index 85b7bafbf1..e0087d94e9 100644
--- a/tools/fmpp-maven-plugin/pom.xml
+++ b/tools/fmpp-maven-plugin/pom.xml
@@ -22,7 +22,7 @@
com.dremio.tools
dremio-tools-parent
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.tools
diff --git a/tools/pom.xml b/tools/pom.xml
index 00154b9622..926c2982c6 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -24,7 +24,7 @@
dremio-root
com.dremio
- 3.1.4-201902210617130445-0d64760
+ 3.1.6-201903070042400578-fdcd3a8
com.dremio.tools