diff --git a/client/jdbc/pom.xml b/client/jdbc/pom.xml index 33f5fd67e3..30b1a343f6 100644 --- a/client/jdbc/pom.xml +++ b/client/jdbc/pom.xml @@ -21,7 +21,7 @@ com.dremio.client dremio-client-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-client-jdbc Client - JDBC Driver diff --git a/client/pom.xml b/client/pom.xml index 9981cfd442..8326b2f895 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.client diff --git a/common/pom.xml b/common/pom.xml index fa6543d5a4..1a01eeb554 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -23,7 +23,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-common diff --git a/contrib/hive-exec-shade/pom.xml b/contrib/hive-exec-shade/pom.xml index 88faa14901..cca74f0d2e 100644 --- a/contrib/hive-exec-shade/pom.xml +++ b/contrib/hive-exec-shade/pom.xml @@ -22,7 +22,7 @@ com.dremio.contrib dremio-contrib-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-hive-exec-shaded diff --git a/contrib/pom.xml b/contrib/pom.xml index 913fa7ce3a..7dafdaf7ef 100644 --- a/contrib/pom.xml +++ b/contrib/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.contrib diff --git a/contrib/slim-shade/pom.xml b/contrib/slim-shade/pom.xml index c13f0ad50e..8753eaf0c8 100644 --- a/contrib/slim-shade/pom.xml +++ b/contrib/slim-shade/pom.xml @@ -22,7 +22,7 @@ com.dremio.contrib dremio-contrib-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-slim-shade diff --git a/contrib/tpch-sample-data/pom.xml b/contrib/tpch-sample-data/pom.xml index bebf9be7af..ed0858798b 100644 --- a/contrib/tpch-sample-data/pom.xml +++ b/contrib/tpch-sample-data/pom.xml @@ -21,7 +21,7 @@ dremio-contrib-parent com.dremio.contrib - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-tpch-sample-data diff --git a/dac/backend/pom.xml b/dac/backend/pom.xml index d88503b1c5..54efc8c651 100644 --- a/dac/backend/pom.xml +++ b/dac/backend/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-dac-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-dac-backend diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java index dc02ee1a0e..26aee1bf78 100644 --- a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java +++ b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileConf.java @@ -121,7 +121,7 @@ public SchemaMutability getSchemaMutability() { @Override public HomeFileSystemStoragePlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) { - return new HomeFileSystemStoragePlugin(this, context, name, null, pluginIdProvider); + return new HomeFileSystemStoragePlugin(this, context, name, pluginIdProvider); } public Path getBaseUploadsPath() { diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java index af8c012d51..bc966eb0b6 100644 --- a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java +++ b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileSystemStoragePlugin.java @@ -66,8 +66,8 @@ public class HomeFileSystemStoragePlugin extends FileSystemPlugin private final Path stagingDir; private final Path uploadsDir; - public HomeFileSystemStoragePlugin(final HomeFileConf config, final SabotContext context, final String name, FileSystemWrapper fs, Provider idProvider) { - super(config, context, name, fs, idProvider); + public HomeFileSystemStoragePlugin(final HomeFileConf config, final SabotContext context, final String name, Provider idProvider) { + super(config, context, name, idProvider); this.stagingDir = new Path(config.getPath(), STAGING + "." + context.getDremioConfig().getThisNode()); this.uploadsDir = new Path(config.getPath(), UPLOADS); } @@ -80,7 +80,7 @@ public HomeFileConf getConfig() { @Override public void start() throws IOException { super.start(); - FileSystem fs = getFs(); + FileSystem fs = getSystemUserFS(); fs.mkdirs(getConfig().getPath(), DEFAULT_PERMISSIONS); fs.mkdirs(stagingDir, DEFAULT_PERMISSIONS); fs.mkdirs(uploadsDir, DEFAULT_PERMISSIONS); @@ -96,7 +96,7 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig PhysicalDataset physicalDataset = oldConfig == null ? null : oldConfig.getPhysicalDataset(); FormatPluginConfig pluginConfig = null; try { - final FileSystemWrapper fs = getFs(); + final FileSystemWrapper fs = getSystemUserFS(); final FileConfig fileConfig; if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ @@ -113,7 +113,7 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig pluginConfig = PhysicalDatasetUtils.toFormatPlugin(fileConfig, Collections.emptyList()); final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(pluginConfig); return getDataset(datasetPath, oldConfig, formatPlugin, fs, fileConfig, - retrievalOptions.maxMetadataLeafColumns()); + retrievalOptions.maxMetadataLeafColumns()); } catch (NamespaceNotFoundException nfe){ FormatPluginConfig formatPluginConfig = null; if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ @@ -125,9 +125,9 @@ public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig @Override protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPluginConfig formatPluginConfig, - DatasetRetrievalOptions retrievalOptions, String user) throws Exception { + DatasetRetrievalOptions retrievalOptions, String user) throws Exception { try{ - final FileSystemWrapper fs = getFs(); + final FileSystemWrapper fs = getSystemUserFS(); final DatasetConfig datasetConfig = getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath); if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) { @@ -137,7 +137,7 @@ protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, D final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(formatPluginConfig); return getDataset(datasetPath, oldConfig, formatPlugin, fs, - datasetConfig.getPhysicalDataset().getFormatSettings(), retrievalOptions.maxMetadataLeafColumns()); + datasetConfig.getPhysicalDataset().getFormatSettings(), retrievalOptions.maxMetadataLeafColumns()); } catch (NamespaceNotFoundException nfe){ if(formatPluginConfig == null) { // a home file can only be read from the namespace or using a format options. Without either, it is invalid, return nothing. @@ -152,10 +152,10 @@ protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, D public boolean containerExists(NamespaceKey key) { List folderPath = key.getPathComponents(); try { - return getFS(SystemUser.SYSTEM_USERNAME).exists(PathUtils.toFSPath( - ImmutableList.builder() - .addAll(folderPath.subList(1, folderPath.size())) - .build())); + return getSystemUserFS().exists(PathUtils.toFSPath( + ImmutableList.builder() + .addAll(folderPath.subList(1, folderPath.size())) + .build())); }catch(IOException e) { logger.info("IOException while trying to retrieve home files.", e); return false; @@ -163,12 +163,12 @@ public boolean containerExists(NamespaceKey key) { } private SourceTableDefinition getDataset( - NamespaceKey datasetPath, - DatasetConfig oldConfig, - FormatPlugin formatPlugin, - FileSystemWrapper fs, - FileConfig fileConfig, - int maxLeafColumns + NamespaceKey datasetPath, + DatasetConfig oldConfig, + FormatPlugin formatPlugin, + FileSystemWrapper fs, + FileConfig fileConfig, + int maxLeafColumns ) throws IOException { final List cachedEntities = Lists.newArrayList(); diff --git a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java index e6d1cae382..37022fe95b 100644 --- a/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java +++ b/dac/backend/src/main/java/com/dremio/dac/homefiles/HomeFileTool.java @@ -55,7 +55,7 @@ public HomeFileTool(SabotContext context, CatalogService catalog) throws Executi StoragePlugin plugin = catalog.getSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME); Preconditions.checkNotNull(plugin, "Plugin [%s] not found.", HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME); HomeFileSystemStoragePlugin homePlugin = (HomeFileSystemStoragePlugin) plugin; - this.fs = homePlugin.getProcessFs(); + this.fs = homePlugin.getSystemUserFS(); this.config = homePlugin.getConfig(); this.hostname = context.getDremioConfig().getThisNode(); } diff --git a/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java b/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java index 6f39d7f6ed..8589f0d812 100644 --- a/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java +++ b/dac/backend/src/main/java/com/dremio/dac/model/sources/FormatTools.java @@ -168,7 +168,7 @@ public FileFormat getOrDetectFormat(NamespacePath folderPath, DatasetType expect private FileFormat detectFileFormat(NamespaceKey key) { final FileSystemPlugin plugin = getPlugin(key); - final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName()); + final FileSystemWrapper fs = plugin.createFS(securityContext.getUserPrincipal().getName()); final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents())); // for now, existing rudimentary behavior that uses extension detection. @@ -240,7 +240,7 @@ private static FileFormat asFormat(NamespaceKey key, Path path, boolean isFolder public JobDataFragment previewData(FileFormat format, NamespacePath namespacePath, boolean useFormatLocation) { final NamespaceKey key = namespacePath.toNamespaceKey(); final FileSystemPlugin plugin = getPlugin(key); - final FileSystemWrapper fs = plugin.getFS(securityContext.getUserPrincipal().getName()); + final FileSystemWrapper fs = plugin.createFS(securityContext.getUserPrincipal().getName()); final Path path = FileSelection.getPathBasedOnFullPath(plugin.resolveTableNameToValidPath(key.getPathComponents())); // for now, existing rudimentary behavior that uses extension detection. @@ -342,7 +342,7 @@ private JobDataFragment getData(FileFormat format, FileSystemPlugin plugin, File if (first) { first = false; container.buildSchema(); - if (container.getSchema().getFieldCount() > maxLeafColumns) { + if (container.getSchema().getTotalFieldCount() > maxLeafColumns) { throw UserException.validationError() .message("Using datasets with more than %d columns is currently disabled.", maxLeafColumns) .build(logger); diff --git a/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java b/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java index 37dac5a4f3..603dbd5343 100644 --- a/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java +++ b/dac/backend/src/main/java/com/dremio/dac/service/datasets/DatasetVersionMutator.java @@ -110,7 +110,8 @@ public DatasetVersionMutator( private DatasetDownloadManager downloadManager() { final FileSystemPlugin downloadPlugin = catalogService.getSource(DATASET_DOWNLOAD_STORAGE_PLUGIN); - return new DatasetDownloadManager(jobsService, namespaceService, downloadPlugin.getConfig().getPath(), downloadPlugin.getFs()); + return new DatasetDownloadManager(jobsService, namespaceService, downloadPlugin.getConfig().getPath(), + downloadPlugin.getSystemUserFS()); } private void validate(DatasetPath path, VirtualDatasetUI ds) { if (ds.getSqlFieldsList() == null || ds.getSqlFieldsList().isEmpty()) { diff --git a/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java b/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java index 2c61216bac..2cad946397 100644 --- a/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java +++ b/dac/backend/src/test/java/com/dremio/dac/service/APrivateSource.java @@ -73,7 +73,7 @@ public SchemaMutability getSchemaMutability() { @Override public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) { - return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider); + return new FileSystemPlugin<>(this, context, name, pluginIdProvider); } diff --git a/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java b/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java index 9634938bcc..000b6d797e 100644 --- a/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java +++ b/dac/backend/src/test/java/com/dremio/dac/service/TestPhysicalDatasets.java @@ -32,8 +32,11 @@ import org.apache.hadoop.fs.Path; import org.junit.Before; import org.junit.Ignore; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import com.dremio.common.exceptions.UserException; import com.dremio.common.util.FileUtils; import com.dremio.common.utils.PathUtils; import com.dremio.dac.explore.model.FileFormatUI; @@ -73,6 +76,8 @@ * Tests to create, update and execute queries on physical datasets.. */ public class TestPhysicalDatasets extends BaseTestServer { + @Rule + public ExpectedException thrown = ExpectedException.none(); @Before public void setup() throws Exception { @@ -294,6 +299,30 @@ public void testSchemaChangeExit() throws Exception { assertEquals(1, data.getColumns().size()); } + @Test + public void testLargeJsonFile() throws Exception { + final JobsService jobsService = l(JobsService.class); + JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder() + .setSqlQuery(createQuery("/datasets/wide_table.json")) + .setQueryType(QueryType.UI_RUN) + .build(), NoOpJobStatusListener.INSTANCE)); + + thrown.expect(UserException.class); + job.getData().truncate(500); + } + + @Test + public void testLargeNestedJsonFile() throws Exception { + final JobsService jobsService = l(JobsService.class); + JobUI job = new JobUI(jobsService.submitJob(JobRequest.newBuilder() + .setSqlQuery(createQuery("/datasets/wide_nested_table.json")) + .setQueryType(QueryType.UI_RUN) + .build(), NoOpJobStatusListener.INSTANCE)); + + thrown.expect(UserException.class); + job.getData().truncate(500); + } + @Test public void testParquetFile() throws Exception { final JobsService jobsService = l(JobsService.class); diff --git a/dac/backend/src/test/resources/datasets/wide_nested_table.json b/dac/backend/src/test/resources/datasets/wide_nested_table.json new file mode 100644 index 0000000000..cd86e7ea2f --- /dev/null +++ b/dac/backend/src/test/resources/datasets/wide_nested_table.json @@ -0,0 +1 @@ +{ "name" : "test_name", "fields": { "column1" : "col", "column2" : "col", "column3" : "col", "column4" : "col", "column5" : "col", "column6" : "col", "column7" : "col", "column8" : "col", "column9" : "col", "column10" : "col", "column11" : "col", "column12" : "col", "column13" : "col", "column14" : "col", "column15" : "col", "column16" : "col", "column17" : "col", "column18" : "col", "column19" : "col", "column20" : "col", "column21" : "col", "column22" : "col", "column23" : "col", "column24" : "col", "column25" : "col", "column26" : "col", "column27" : "col", "column28" : "col", "column29" : "col", "column30" : "col", "column31" : "col", "column32" : "col", "column33" : "col", "column34" : "col", "column35" : "col", "column36" : "col", "column37" : "col", "column38" : "col", "column39" : "col", "column40" : "col", "column41" : "col", "column42" : "col", "column43" : "col", "column44" : "col", "column45" : "col", "column46" : "col", "column47" : "col", "column48" : "col", "column49" : "col", "column50" : "col", "column51" : "col", "column52" : "col", "column53" : "col", "column54" : "col", "column55" : "col", "column56" : "col", "column57" : "col", "column58" : "col", "column59" : "col", "column60" : "col", "column61" : "col", "column62" : "col", "column63" : "col", "column64" : "col", "column65" : "col", "column66" : "col", "column67" : "col", "column68" : "col", "column69" : "col", "column70" : "col", "column71" : "col", "column72" : "col", "column73" : "col", "column74" : "col", "column75" : "col", "column76" : "col", "column77" : "col", "column78" : "col", "column79" : "col", "column80" : "col", "column81" : "col", "column82" : "col", "column83" : "col", "column84" : "col", "column85" : "col", "column86" : "col", "column87" : "col", "column88" : "col", "column89" : "col", "column90" : "col", "column91" : "col", "column92" : "col", "column93" : "col", "column94" : "col", "column95" : "col", "column96" : "col", "column97" : "col", "column98" : "col", "column99" : "col", "column100" : "col" }, "more_fields": { "local_fields": { "column101" : "col", "column102" : "col", "column103" : "col", "column104" : "col", "column105" : "col", "column106" : "col", "column107" : "col", "column108" : "col", "column109" : "col", "column110" : "col", "column111" : "col", "column112" : "col", "column113" : "col", "column114" : "col", "column115" : "col", "column116" : "col", "column117" : "col", "column118" : "col", "column119" : "col", "column120" : "col", "column121" : "col", "column122" : "col", "column123" : "col", "column124" : "col", "column125" : "col", "column126" : "col", "column127" : "col", "column128" : "col", "column129" : "col", "column130" : "col", "column131" : "col", "column132" : "col", "column133" : "col", "column134" : "col", "column135" : "col", "column136" : "col", "column137" : "col", "column138" : "col", "column139" : "col", "column140" : "col", "column141" : "col", "column142" : "col", "column143" : "col", "column144" : "col", "column145" : "col", "column146" : "col", "column147" : "col", "column148" : "col", "column149" : "col", "column150" : "col", "column151" : "col", "column152" : "col", "column153" : "col", "column154" : "col", "column155" : "col", "column156" : "col", "column157" : "col", "column158" : "col", "column159" : "col", "column160" : "col", "column161" : "col", "column162" : "col", "column163" : "col", "column164" : "col", "column165" : "col", "column166" : "col", "column167" : "col", "column168" : "col", "column169" : "col", "column170" : "col", "column171" : "col", "column172" : "col", "column173" : "col", "column174" : "col", "column175" : "col", "column176" : "col", "column177" : "col", "column178" : "col", "column179" : "col", "column180" : "col", "column181" : "col", "column182" : "col", "column183" : "col", "column184" : "col", "column185" : "col", "column186" : "col", "column187" : "col", "column188" : "col", "column189" : "col", "column190" : "col", "column191" : "col", "column192" : "col", "column193" : "col", "column194" : "col", "column195" : "col", "column196" : "col", "column197" : "col", "column198" : "col", "column199" : "col", "column200" : "col", "column201" : "col", "column202" : "col", "column203" : "col", "column204" : "col", "column205" : "col", "column206" : "col", "column207" : "col", "column208" : "col", "column209" : "col", "column210" : "col", "column211" : "col", "column212" : "col", "column213" : "col", "column214" : "col", "column215" : "col", "column216" : "col", "column217" : "col", "column218" : "col", "column219" : "col", "column220" : "col", "column221" : "col", "column222" : "col", "column223" : "col", "column224" : "col", "column225" : "col", "column226" : "col", "column227" : "col", "column228" : "col", "column229" : "col", "column230" : "col", "column231" : "col", "column232" : "col", "column233" : "col", "column234" : "col", "column235" : "col", "column236" : "col", "column237" : "col", "column238" : "col", "column239" : "col", "column240" : "col", "column241" : "col", "column242" : "col", "column243" : "col", "column244" : "col", "column245" : "col", "column246" : "col", "column247" : "col", "column248" : "col", "column249" : "col", "column250" : "col", "column251" : "col", "column252" : "col", "column253" : "col", "column254" : "col", "column255" : "col", "column256" : "col", "column257" : "col", "column258" : "col", "column259" : "col", "column260" : "col", "column261" : "col", "column262" : "col", "column263" : "col", "column264" : "col", "column265" : "col", "column266" : "col", "column267" : "col", "column268" : "col", "column269" : "col", "column270" : "col", "column271" : "col", "column272" : "col", "column273" : "col", "column274" : "col", "column275" : "col", "column276" : "col", "column277" : "col", "column278" : "col", "column279" : "col", "column280" : "col", "column281" : "col", "column282" : "col", "column283" : "col", "column284" : "col", "column285" : "col", "column286" : "col", "column287" : "col", "column288" : "col", "column289" : "col", "column290" : "col", "column291" : "col", "column292" : "col", "column293" : "col", "column294" : "col", "column295" : "col", "column296" : "col", "column297" : "col", "column298" : "col", "column299" : "col", "column300" : "col", "column301" : "col", "column302" : "col", "column303" : "col", "column304" : "col", "column305" : "col", "column306" : "col", "column307" : "col", "column308" : "col", "column309" : "col", "column310" : "col", "column311" : "col", "column312" : "col", "column313" : "col", "column314" : "col", "column315" : "col", "column316" : "col", "column317" : "col", "column318" : "col", "column319" : "col", "column320" : "col", "column321" : "col", "column322" : "col", "column323" : "col", "column324" : "col", "column325" : "col", "column326" : "col", "column327" : "col", "column328" : "col", "column329" : "col", "column330" : "col", "column331" : "col", "column332" : "col", "column333" : "col", "column334" : "col", "column335" : "col", "column336" : "col", "column337" : "col", "column338" : "col", "column339" : "col", "column340" : "col", "column341" : "col", "column342" : "col", "column343" : "col", "column344" : "col", "column345" : "col", "column346" : "col", "column347" : "col", "column348" : "col", "column349" : "col", "column350" : "col", "column351" : "col", "column352" : "col", "column353" : "col", "column354" : "col", "column355" : "col", "column356" : "col", "column357" : "col", "column358" : "col", "column359" : "col", "column360" : "col", "column361" : "col", "column362" : "col", "column363" : "col", "column364" : "col", "column365" : "col", "column366" : "col", "column367" : "col", "column368" : "col", "column369" : "col", "column370" : "col", "column371" : "col", "column372" : "col", "column373" : "col", "column374" : "col", "column375" : "col", "column376" : "col", "column377" : "col", "column378" : "col", "column379" : "col", "column380" : "col", "column381" : "col", "column382" : "col", "column383" : "col", "column384" : "col", "column385" : "col", "column386" : "col", "column387" : "col", "column388" : "col", "column389" : "col", "column390" : "col", "column391" : "col", "column392" : "col", "column393" : "col", "column394" : "col", "column395" : "col", "column396" : "col", "column397" : "col", "column398" : "col", "column399" : "col" } }, "column400" : "col", "column401" : "col", "column402" : "col", "column403" : "col", "column404" : "col", "column405" : "col", "column406" : "col", "column407" : "col", "column408" : "col", "column409" : "col", "column410" : "col", "column411" : "col", "column412" : "col", "column413" : "col", "column414" : "col", "column415" : "col", "column416" : "col", "column417" : "col", "column418" : "col", "column419" : "col", "column420" : "col", "column421" : "col", "column422" : "col", "column423" : "col", "column424" : "col", "column425" : "col", "column426" : "col", "column427" : "col", "column428" : "col", "column429" : "col", "column430" : "col", "column431" : "col", "column432" : "col", "column433" : "col", "column434" : "col", "column435" : "col", "column436" : "col", "column437" : "col", "column438" : "col", "column439" : "col", "column440" : "col", "column441" : "col", "column442" : "col", "column443" : "col", "column444" : "col", "column445" : "col", "column446" : "col", "column447" : "col", "column448" : "col", "column449" : "col", "column450" : "col", "column451" : "col", "column452" : "col", "column453" : "col", "column454" : "col", "column455" : "col", "column456" : "col", "column457" : "col", "column458" : "col", "column459" : "col", "column460" : "col", "column461" : "col", "column462" : "col", "column463" : "col", "column464" : "col", "column465" : "col", "column466" : "col", "column467" : "col", "column468" : "col", "column469" : "col", "column470" : "col", "column471" : "col", "column472" : "col", "column473" : "col", "column474" : "col", "column475" : "col", "column476" : "col", "column477" : "col", "column478" : "col", "column479" : "col", "column480" : "col", "column481" : "col", "column482" : "col", "column483" : "col", "column484" : "col", "column485" : "col", "column486" : "col", "column487" : "col", "column488" : "col", "column489" : "col", "column490" : "col", "column491" : "col", "column492" : "col", "column493" : "col", "column494" : "col", "column495" : "col", "column496" : "col", "column497" : "col", "column498" : "col", "column499" : "col", "column500" : "col", "column501" : "col", "column502" : "col", "column503" : "col", "column504" : "col", "column505" : "col", "column506" : "col", "column507" : "col", "column508" : "col", "column509" : "col", "column510" : "col", "column511" : "col", "column512" : "col", "column513" : "col", "column514" : "col", "column515" : "col", "column516" : "col", "column517" : "col", "column518" : "col", "column519" : "col", "column520" : "col", "column521" : "col", "column522" : "col", "column523" : "col", "column524" : "col", "column525" : "col", "column526" : "col", "column527" : "col", "column528" : "col", "column529" : "col", "column530" : "col", "column531" : "col", "column532" : "col", "column533" : "col", "column534" : "col", "column535" : "col", "column536" : "col", "column537" : "col", "column538" : "col", "column539" : "col", "column540" : "col", "column541" : "col", "column542" : "col", "column543" : "col", "column544" : "col", "column545" : "col", "column546" : "col", "column547" : "col", "column548" : "col", "column549" : "col", "column550" : "col", "column551" : "col", "column552" : "col", "column553" : "col", "column554" : "col", "column555" : "col", "column556" : "col", "column557" : "col", "column558" : "col", "column559" : "col", "column560" : "col", "column561" : "col", "column562" : "col", "column563" : "col", "column564" : "col", "column565" : "col", "column566" : "col", "column567" : "col", "column568" : "col", "column569" : "col", "column570" : "col", "column571" : "col", "column572" : "col", "column573" : "col", "column574" : "col", "column575" : "col", "column576" : "col", "column577" : "col", "column578" : "col", "column579" : "col", "column580" : "col", "column581" : "col", "column582" : "col", "column583" : "col", "column584" : "col", "column585" : "col", "column586" : "col", "column587" : "col", "column588" : "col", "column589" : "col", "column590" : "col", "column591" : "col", "column592" : "col", "column593" : "col", "column594" : "col", "column595" : "col", "column596" : "col", "column597" : "col", "column598" : "col", "column599" : "col", "column600" : "col", "column601" : "col", "column602" : "col", "column603" : "col", "column604" : "col", "column605" : "col", "column606" : "col", "column607" : "col", "column608" : "col", "column609" : "col", "column610" : "col", "column611" : "col", "column612" : "col", "column613" : "col", "column614" : "col", "column615" : "col", "column616" : "col", "column617" : "col", "column618" : "col", "column619" : "col", "column620" : "col", "column621" : "col", "column622" : "col", "column623" : "col", "column624" : "col", "column625" : "col", "column626" : "col", "column627" : "col", "column628" : "col", "column629" : "col", "column630" : "col", "column631" : "col", "column632" : "col", "column633" : "col", "column634" : "col", "column635" : "col", "column636" : "col", "column637" : "col", "column638" : "col", "column639" : "col", "column640" : "col", "column641" : "col", "column642" : "col", "column643" : "col", "column644" : "col", "column645" : "col", "column646" : "col", "column647" : "col", "column648" : "col", "column649" : "col", "column650" : "col", "column651" : "col", "column652" : "col", "column653" : "col", "column654" : "col", "column655" : "col", "column656" : "col", "column657" : "col", "column658" : "col", "column659" : "col", "column660" : "col", "column661" : "col", "column662" : "col", "column663" : "col", "column664" : "col", "column665" : "col", "column666" : "col", "column667" : "col", "column668" : "col", "column669" : "col", "column670" : "col", "column671" : "col", "column672" : "col", "column673" : "col", "column674" : "col", "column675" : "col", "column676" : "col", "column677" : "col", "column678" : "col", "column679" : "col", "column680" : "col", "column681" : "col", "column682" : "col", "column683" : "col", "column684" : "col", "column685" : "col", "column686" : "col", "column687" : "col", "column688" : "col", "column689" : "col", "column690" : "col", "column691" : "col", "column692" : "col", "column693" : "col", "column694" : "col", "column695" : "col", "column696" : "col", "column697" : "col", "column698" : "col", "column699" : "col", "column700" : "col", "column701" : "col", "column702" : "col", "column703" : "col", "column704" : "col", "column705" : "col", "column706" : "col", "column707" : "col", "column708" : "col", "column709" : "col", "column710" : "col", "column711" : "col", "column712" : "col", "column713" : "col", "column714" : "col", "column715" : "col", "column716" : "col", "column717" : "col", "column718" : "col", "column719" : "col", "column720" : "col", "column721" : "col", "column722" : "col", "column723" : "col", "column724" : "col", "column725" : "col", "column726" : "col", "column727" : "col", "column728" : "col", "column729" : "col", "column730" : "col", "column731" : "col", "column732" : "col", "column733" : "col", "column734" : "col", "column735" : "col", "column736" : "col", "column737" : "col", "column738" : "col", "column739" : "col", "column740" : "col", "column741" : "col", "column742" : "col", "column743" : "col", "column744" : "col", "column745" : "col", "column746" : "col", "column747" : "col", "column748" : "col", "column749" : "col", "column750" : "col", "column751" : "col", "column752" : "col", "column753" : "col", "column754" : "col", "column755" : "col", "column756" : "col", "column757" : "col", "column758" : "col", "column759" : "col", "column760" : "col", "column761" : "col", "column762" : "col", "column763" : "col", "column764" : "col", "column765" : "col", "column766" : "col", "column767" : "col", "column768" : "col", "column769" : "col", "column770" : "col", "column771" : "col", "column772" : "col", "column773" : "col", "column774" : "col", "column775" : "col", "column776" : "col", "column777" : "col", "column778" : "col", "column779" : "col", "column780" : "col", "column781" : "col", "column782" : "col", "column783" : "col", "column784" : "col", "column785" : "col", "column786" : "col", "column787" : "col", "column788" : "col", "column789" : "col", "column790" : "col", "column791" : "col", "column792" : "col", "column793" : "col", "column794" : "col", "column795" : "col", "column796" : "col", "column797" : "col", "column798" : "col", "column799" : "col", "column800" : "col", "column801" : "col", "column802" : "col", "column803" : "col", "column804" : "col", "column805" : "col", "column806" : "col", "column807" : "col", "column808" : "col", "column809" : "col", "column810" : "col" } diff --git a/dac/backend/src/test/resources/datasets/wide_table.json b/dac/backend/src/test/resources/datasets/wide_table.json new file mode 100644 index 0000000000..0df384a674 --- /dev/null +++ b/dac/backend/src/test/resources/datasets/wide_table.json @@ -0,0 +1 @@ +{ "column1" : "col", "column2" : "col", "column3" : "col", "column4" : "col", "column5" : "col", "column6" : "col", "column7" : "col", "column8" : "col", "column9" : "col", "column10" : "col", "column11" : "col", "column12" : "col", "column13" : "col", "column14" : "col", "column15" : "col", "column16" : "col", "column17" : "col", "column18" : "col", "column19" : "col", "column20" : "col", "column21" : "col", "column22" : "col", "column23" : "col", "column24" : "col", "column25" : "col", "column26" : "col", "column27" : "col", "column28" : "col", "column29" : "col", "column30" : "col", "column31" : "col", "column32" : "col", "column33" : "col", "column34" : "col", "column35" : "col", "column36" : "col", "column37" : "col", "column38" : "col", "column39" : "col", "column40" : "col", "column41" : "col", "column42" : "col", "column43" : "col", "column44" : "col", "column45" : "col", "column46" : "col", "column47" : "col", "column48" : "col", "column49" : "col", "column50" : "col", "column51" : "col", "column52" : "col", "column53" : "col", "column54" : "col", "column55" : "col", "column56" : "col", "column57" : "col", "column58" : "col", "column59" : "col", "column60" : "col", "column61" : "col", "column62" : "col", "column63" : "col", "column64" : "col", "column65" : "col", "column66" : "col", "column67" : "col", "column68" : "col", "column69" : "col", "column70" : "col", "column71" : "col", "column72" : "col", "column73" : "col", "column74" : "col", "column75" : "col", "column76" : "col", "column77" : "col", "column78" : "col", "column79" : "col", "column80" : "col", "column81" : "col", "column82" : "col", "column83" : "col", "column84" : "col", "column85" : "col", "column86" : "col", "column87" : "col", "column88" : "col", "column89" : "col", "column90" : "col", "column91" : "col", "column92" : "col", "column93" : "col", "column94" : "col", "column95" : "col", "column96" : "col", "column97" : "col", "column98" : "col", "column99" : "col", "column100" : "col", "column101" : "col", "column102" : "col", "column103" : "col", "column104" : "col", "column105" : "col", "column106" : "col", "column107" : "col", "column108" : "col", "column109" : "col", "column110" : "col", "column111" : "col", "column112" : "col", "column113" : "col", "column114" : "col", "column115" : "col", "column116" : "col", "column117" : "col", "column118" : "col", "column119" : "col", "column120" : "col", "column121" : "col", "column122" : "col", "column123" : "col", "column124" : "col", "column125" : "col", "column126" : "col", "column127" : "col", "column128" : "col", "column129" : "col", "column130" : "col", "column131" : "col", "column132" : "col", "column133" : "col", "column134" : "col", "column135" : "col", "column136" : "col", "column137" : "col", "column138" : "col", "column139" : "col", "column140" : "col", "column141" : "col", "column142" : "col", "column143" : "col", "column144" : "col", "column145" : "col", "column146" : "col", "column147" : "col", "column148" : "col", "column149" : "col", "column150" : "col", "column151" : "col", "column152" : "col", "column153" : "col", "column154" : "col", "column155" : "col", "column156" : "col", "column157" : "col", "column158" : "col", "column159" : "col", "column160" : "col", "column161" : "col", "column162" : "col", "column163" : "col", "column164" : "col", "column165" : "col", "column166" : "col", "column167" : "col", "column168" : "col", "column169" : "col", "column170" : "col", "column171" : "col", "column172" : "col", "column173" : "col", "column174" : "col", "column175" : "col", "column176" : "col", "column177" : "col", "column178" : "col", "column179" : "col", "column180" : "col", "column181" : "col", "column182" : "col", "column183" : "col", "column184" : "col", "column185" : "col", "column186" : "col", "column187" : "col", "column188" : "col", "column189" : "col", "column190" : "col", "column191" : "col", "column192" : "col", "column193" : "col", "column194" : "col", "column195" : "col", "column196" : "col", "column197" : "col", "column198" : "col", "column199" : "col", "column200" : "col", "column201" : "col", "column202" : "col", "column203" : "col", "column204" : "col", "column205" : "col", "column206" : "col", "column207" : "col", "column208" : "col", "column209" : "col", "column210" : "col", "column211" : "col", "column212" : "col", "column213" : "col", "column214" : "col", "column215" : "col", "column216" : "col", "column217" : "col", "column218" : "col", "column219" : "col", "column220" : "col", "column221" : "col", "column222" : "col", "column223" : "col", "column224" : "col", "column225" : "col", "column226" : "col", "column227" : "col", "column228" : "col", "column229" : "col", "column230" : "col", "column231" : "col", "column232" : "col", "column233" : "col", "column234" : "col", "column235" : "col", "column236" : "col", "column237" : "col", "column238" : "col", "column239" : "col", "column240" : "col", "column241" : "col", "column242" : "col", "column243" : "col", "column244" : "col", "column245" : "col", "column246" : "col", "column247" : "col", "column248" : "col", "column249" : "col", "column250" : "col", "column251" : "col", "column252" : "col", "column253" : "col", "column254" : "col", "column255" : "col", "column256" : "col", "column257" : "col", "column258" : "col", "column259" : "col", "column260" : "col", "column261" : "col", "column262" : "col", "column263" : "col", "column264" : "col", "column265" : "col", "column266" : "col", "column267" : "col", "column268" : "col", "column269" : "col", "column270" : "col", "column271" : "col", "column272" : "col", "column273" : "col", "column274" : "col", "column275" : "col", "column276" : "col", "column277" : "col", "column278" : "col", "column279" : "col", "column280" : "col", "column281" : "col", "column282" : "col", "column283" : "col", "column284" : "col", "column285" : "col", "column286" : "col", "column287" : "col", "column288" : "col", "column289" : "col", "column290" : "col", "column291" : "col", "column292" : "col", "column293" : "col", "column294" : "col", "column295" : "col", "column296" : "col", "column297" : "col", "column298" : "col", "column299" : "col", "column300" : "col", "column301" : "col", "column302" : "col", "column303" : "col", "column304" : "col", "column305" : "col", "column306" : "col", "column307" : "col", "column308" : "col", "column309" : "col", "column310" : "col", "column311" : "col", "column312" : "col", "column313" : "col", "column314" : "col", "column315" : "col", "column316" : "col", "column317" : "col", "column318" : "col", "column319" : "col", "column320" : "col", "column321" : "col", "column322" : "col", "column323" : "col", "column324" : "col", "column325" : "col", "column326" : "col", "column327" : "col", "column328" : "col", "column329" : "col", "column330" : "col", "column331" : "col", "column332" : "col", "column333" : "col", "column334" : "col", "column335" : "col", "column336" : "col", "column337" : "col", "column338" : "col", "column339" : "col", "column340" : "col", "column341" : "col", "column342" : "col", "column343" : "col", "column344" : "col", "column345" : "col", "column346" : "col", "column347" : "col", "column348" : "col", "column349" : "col", "column350" : "col", "column351" : "col", "column352" : "col", "column353" : "col", "column354" : "col", "column355" : "col", "column356" : "col", "column357" : "col", "column358" : "col", "column359" : "col", "column360" : "col", "column361" : "col", "column362" : "col", "column363" : "col", "column364" : "col", "column365" : "col", "column366" : "col", "column367" : "col", "column368" : "col", "column369" : "col", "column370" : "col", "column371" : "col", "column372" : "col", "column373" : "col", "column374" : "col", "column375" : "col", "column376" : "col", "column377" : "col", "column378" : "col", "column379" : "col", "column380" : "col", "column381" : "col", "column382" : "col", "column383" : "col", "column384" : "col", "column385" : "col", "column386" : "col", "column387" : "col", "column388" : "col", "column389" : "col", "column390" : "col", "column391" : "col", "column392" : "col", "column393" : "col", "column394" : "col", "column395" : "col", "column396" : "col", "column397" : "col", "column398" : "col", "column399" : "col", "column400" : "col", "column401" : "col", "column402" : "col", "column403" : "col", "column404" : "col", "column405" : "col", "column406" : "col", "column407" : "col", "column408" : "col", "column409" : "col", "column410" : "col", "column411" : "col", "column412" : "col", "column413" : "col", "column414" : "col", "column415" : "col", "column416" : "col", "column417" : "col", "column418" : "col", "column419" : "col", "column420" : "col", "column421" : "col", "column422" : "col", "column423" : "col", "column424" : "col", "column425" : "col", "column426" : "col", "column427" : "col", "column428" : "col", "column429" : "col", "column430" : "col", "column431" : "col", "column432" : "col", "column433" : "col", "column434" : "col", "column435" : "col", "column436" : "col", "column437" : "col", "column438" : "col", "column439" : "col", "column440" : "col", "column441" : "col", "column442" : "col", "column443" : "col", "column444" : "col", "column445" : "col", "column446" : "col", "column447" : "col", "column448" : "col", "column449" : "col", "column450" : "col", "column451" : "col", "column452" : "col", "column453" : "col", "column454" : "col", "column455" : "col", "column456" : "col", "column457" : "col", "column458" : "col", "column459" : "col", "column460" : "col", "column461" : "col", "column462" : "col", "column463" : "col", "column464" : "col", "column465" : "col", "column466" : "col", "column467" : "col", "column468" : "col", "column469" : "col", "column470" : "col", "column471" : "col", "column472" : "col", "column473" : "col", "column474" : "col", "column475" : "col", "column476" : "col", "column477" : "col", "column478" : "col", "column479" : "col", "column480" : "col", "column481" : "col", "column482" : "col", "column483" : "col", "column484" : "col", "column485" : "col", "column486" : "col", "column487" : "col", "column488" : "col", "column489" : "col", "column490" : "col", "column491" : "col", "column492" : "col", "column493" : "col", "column494" : "col", "column495" : "col", "column496" : "col", "column497" : "col", "column498" : "col", "column499" : "col", "column500" : "col", "column501" : "col", "column502" : "col", "column503" : "col", "column504" : "col", "column505" : "col", "column506" : "col", "column507" : "col", "column508" : "col", "column509" : "col", "column510" : "col", "column511" : "col", "column512" : "col", "column513" : "col", "column514" : "col", "column515" : "col", "column516" : "col", "column517" : "col", "column518" : "col", "column519" : "col", "column520" : "col", "column521" : "col", "column522" : "col", "column523" : "col", "column524" : "col", "column525" : "col", "column526" : "col", "column527" : "col", "column528" : "col", "column529" : "col", "column530" : "col", "column531" : "col", "column532" : "col", "column533" : "col", "column534" : "col", "column535" : "col", "column536" : "col", "column537" : "col", "column538" : "col", "column539" : "col", "column540" : "col", "column541" : "col", "column542" : "col", "column543" : "col", "column544" : "col", "column545" : "col", "column546" : "col", "column547" : "col", "column548" : "col", "column549" : "col", "column550" : "col", "column551" : "col", "column552" : "col", "column553" : "col", "column554" : "col", "column555" : "col", "column556" : "col", "column557" : "col", "column558" : "col", "column559" : "col", "column560" : "col", "column561" : "col", "column562" : "col", "column563" : "col", "column564" : "col", "column565" : "col", "column566" : "col", "column567" : "col", "column568" : "col", "column569" : "col", "column570" : "col", "column571" : "col", "column572" : "col", "column573" : "col", "column574" : "col", "column575" : "col", "column576" : "col", "column577" : "col", "column578" : "col", "column579" : "col", "column580" : "col", "column581" : "col", "column582" : "col", "column583" : "col", "column584" : "col", "column585" : "col", "column586" : "col", "column587" : "col", "column588" : "col", "column589" : "col", "column590" : "col", "column591" : "col", "column592" : "col", "column593" : "col", "column594" : "col", "column595" : "col", "column596" : "col", "column597" : "col", "column598" : "col", "column599" : "col", "column600" : "col", "column601" : "col", "column602" : "col", "column603" : "col", "column604" : "col", "column605" : "col", "column606" : "col", "column607" : "col", "column608" : "col", "column609" : "col", "column610" : "col", "column611" : "col", "column612" : "col", "column613" : "col", "column614" : "col", "column615" : "col", "column616" : "col", "column617" : "col", "column618" : "col", "column619" : "col", "column620" : "col", "column621" : "col", "column622" : "col", "column623" : "col", "column624" : "col", "column625" : "col", "column626" : "col", "column627" : "col", "column628" : "col", "column629" : "col", "column630" : "col", "column631" : "col", "column632" : "col", "column633" : "col", "column634" : "col", "column635" : "col", "column636" : "col", "column637" : "col", "column638" : "col", "column639" : "col", "column640" : "col", "column641" : "col", "column642" : "col", "column643" : "col", "column644" : "col", "column645" : "col", "column646" : "col", "column647" : "col", "column648" : "col", "column649" : "col", "column650" : "col", "column651" : "col", "column652" : "col", "column653" : "col", "column654" : "col", "column655" : "col", "column656" : "col", "column657" : "col", "column658" : "col", "column659" : "col", "column660" : "col", "column661" : "col", "column662" : "col", "column663" : "col", "column664" : "col", "column665" : "col", "column666" : "col", "column667" : "col", "column668" : "col", "column669" : "col", "column670" : "col", "column671" : "col", "column672" : "col", "column673" : "col", "column674" : "col", "column675" : "col", "column676" : "col", "column677" : "col", "column678" : "col", "column679" : "col", "column680" : "col", "column681" : "col", "column682" : "col", "column683" : "col", "column684" : "col", "column685" : "col", "column686" : "col", "column687" : "col", "column688" : "col", "column689" : "col", "column690" : "col", "column691" : "col", "column692" : "col", "column693" : "col", "column694" : "col", "column695" : "col", "column696" : "col", "column697" : "col", "column698" : "col", "column699" : "col", "column700" : "col", "column701" : "col", "column702" : "col", "column703" : "col", "column704" : "col", "column705" : "col", "column706" : "col", "column707" : "col", "column708" : "col", "column709" : "col", "column710" : "col", "column711" : "col", "column712" : "col", "column713" : "col", "column714" : "col", "column715" : "col", "column716" : "col", "column717" : "col", "column718" : "col", "column719" : "col", "column720" : "col", "column721" : "col", "column722" : "col", "column723" : "col", "column724" : "col", "column725" : "col", "column726" : "col", "column727" : "col", "column728" : "col", "column729" : "col", "column730" : "col", "column731" : "col", "column732" : "col", "column733" : "col", "column734" : "col", "column735" : "col", "column736" : "col", "column737" : "col", "column738" : "col", "column739" : "col", "column740" : "col", "column741" : "col", "column742" : "col", "column743" : "col", "column744" : "col", "column745" : "col", "column746" : "col", "column747" : "col", "column748" : "col", "column749" : "col", "column750" : "col", "column751" : "col", "column752" : "col", "column753" : "col", "column754" : "col", "column755" : "col", "column756" : "col", "column757" : "col", "column758" : "col", "column759" : "col", "column760" : "col", "column761" : "col", "column762" : "col", "column763" : "col", "column764" : "col", "column765" : "col", "column766" : "col", "column767" : "col", "column768" : "col", "column769" : "col", "column770" : "col", "column771" : "col", "column772" : "col", "column773" : "col", "column774" : "col", "column775" : "col", "column776" : "col", "column777" : "col", "column778" : "col", "column779" : "col", "column780" : "col", "column781" : "col", "column782" : "col", "column783" : "col", "column784" : "col", "column785" : "col", "column786" : "col", "column787" : "col", "column788" : "col", "column789" : "col", "column790" : "col", "column791" : "col", "column792" : "col", "column793" : "col", "column794" : "col", "column795" : "col", "column796" : "col", "column797" : "col", "column798" : "col", "column799" : "col", "column800" : "col", "column801" : "col", "column802" : "col", "column803" : "col", "column804" : "col", "column805" : "col", "column806" : "col", "column807" : "col", "column808" : "col", "column809" : "col", "column810" : "col" } diff --git a/dac/common/pom.xml b/dac/common/pom.xml index 57f6b9070a..b7f5f1f4b6 100644 --- a/dac/common/pom.xml +++ b/dac/common/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-dac-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-dac-common diff --git a/dac/daemon/pom.xml b/dac/daemon/pom.xml index 7b81dd20b2..d743280e1a 100644 --- a/dac/daemon/pom.xml +++ b/dac/daemon/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-dac-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-dac-daemon diff --git a/dac/pom.xml b/dac/pom.xml index b791a598ba..d6518ca090 100644 --- a/dac/pom.xml +++ b/dac/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-dac-parent diff --git a/dac/ui/pom.xml b/dac/ui/pom.xml index 97bafae1a3..0603ed70d7 100644 --- a/dac/ui/pom.xml +++ b/dac/ui/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-dac-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-dac-ui diff --git a/distribution/jdbc-driver/pom.xml b/distribution/jdbc-driver/pom.xml index 67ec216cc4..25e875cf07 100644 --- a/distribution/jdbc-driver/pom.xml +++ b/distribution/jdbc-driver/pom.xml @@ -21,7 +21,7 @@ com.dremio.distribution dremio-distribution-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-jdbc-driver diff --git a/distribution/pom.xml b/distribution/pom.xml index c23052a5d4..32499fc545 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.distribution diff --git a/distribution/resources/pom.xml b/distribution/resources/pom.xml index d728a4f829..de126de9d5 100644 --- a/distribution/resources/pom.xml +++ b/distribution/resources/pom.xml @@ -22,7 +22,7 @@ com.dremio.distribution dremio-distribution-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-distribution-resources diff --git a/distribution/resources/src/main/resources/bin/dremio b/distribution/resources/src/main/resources/bin/dremio index cf358f4c85..f065d06b32 100755 --- a/distribution/resources/src/main/resources/bin/dremio +++ b/distribution/resources/src/main/resources/bin/dremio @@ -55,6 +55,15 @@ shift command=dremio +print_warn() { + echo "#######################################################################" + echo "WARNING:" + echo "" + echo -e "$1" + echo "" + echo "#######################################################################" +} + wait_for_process_end() { pidKilled=$1 commandName=$2 @@ -188,13 +197,22 @@ DREMIO_NICENESS="${DREMIO_NICENESS:-0}" thiscmd=$0 args=$@ +ULIMIT=65536 +if [ "$(ulimit -n)" -lt "$ULIMIT" ]; then + print_warn "\ +Please increase the maximum number of file descriptors to at least $ULIMIT.\n\n\ +Dremio utilizes a large number of file descriptors.\n\ +The maximum number of file descriptors, currently set at $(ulimit -n), is not sufficient." +fi + case $startStopStatus in (start) check_before_start - echo starting $command, logging to $logout + echo Starting $command, logging to $logout nohup $thiscmd internal_start $args < /dev/null >> ${logout} 2>&1 & sleep 1; + echo Check the log file to make sure $command started successfully and did not run into any issues. ;; (start-fg) diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index 357d98d5f8..ac50478056 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -22,7 +22,7 @@ com.dremio.distribution dremio-distribution-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-distribution diff --git a/plugins/elasticsearch/pom.xml b/plugins/elasticsearch/pom.xml index bc5156c8dc..d9baf6e11f 100644 --- a/plugins/elasticsearch/pom.xml +++ b/plugins/elasticsearch/pom.xml @@ -26,7 +26,7 @@ dremio-plugin-parent com.dremio.plugin - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-elasticsearch-plugin diff --git a/plugins/hbase/pom.xml b/plugins/hbase/pom.xml index 236cfec2df..561dc565ba 100644 --- a/plugins/hbase/pom.xml +++ b/plugins/hbase/pom.xml @@ -22,7 +22,7 @@ com.dremio.plugin dremio-plugin-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-hbase-plugin diff --git a/plugins/hive/pom.xml b/plugins/hive/pom.xml index b32f395b80..e27e17afe0 100644 --- a/plugins/hive/pom.xml +++ b/plugins/hive/pom.xml @@ -22,7 +22,7 @@ com.dremio.plugin dremio-plugin-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-hive-plugin diff --git a/plugins/jdbc/pom.xml b/plugins/jdbc/pom.xml index 1cd3f22bf4..81b826e084 100755 --- a/plugins/jdbc/pom.xml +++ b/plugins/jdbc/pom.xml @@ -22,7 +22,7 @@ com.dremio.plugin dremio-plugin-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-jdbc-plugin diff --git a/plugins/mongo/pom.xml b/plugins/mongo/pom.xml index 505ec208f3..8fd891d5bf 100644 --- a/plugins/mongo/pom.xml +++ b/plugins/mongo/pom.xml @@ -26,7 +26,7 @@ dremio-plugin-parent com.dremio.plugin - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-mongo-plugin diff --git a/plugins/pdfs/pom.xml b/plugins/pdfs/pom.xml index 2c00e1f6ab..98817fe656 100644 --- a/plugins/pdfs/pom.xml +++ b/plugins/pdfs/pom.xml @@ -25,7 +25,7 @@ dremio-plugin-parent com.dremio.plugin - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-pdfs-plugin diff --git a/plugins/pdfs/src/main/java/com/dremio/exec/store/dfs/PDFSConf.java b/plugins/pdfs/src/main/java/com/dremio/exec/store/dfs/PDFSConf.java index 2628d90923..c019bdfb8c 100644 --- a/plugins/pdfs/src/main/java/com/dremio/exec/store/dfs/PDFSConf.java +++ b/plugins/pdfs/src/main/java/com/dremio/exec/store/dfs/PDFSConf.java @@ -65,7 +65,7 @@ public SchemaMutability getSchemaMutability() { @Override public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) { - return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider); + return new FileSystemPlugin<>(this, context, name, pluginIdProvider); } } diff --git a/plugins/pom.xml b/plugins/pom.xml index b89eb9c78b..892420efe6 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -24,7 +24,7 @@ dremio-root com.dremio - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.plugin diff --git a/plugins/s3/pom.xml b/plugins/s3/pom.xml index b50395aac5..eacc55f5b9 100644 --- a/plugins/s3/pom.xml +++ b/plugins/s3/pom.xml @@ -26,7 +26,7 @@ dremio-plugin-parent com.dremio.plugin - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-s3-plugin diff --git a/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeConf.java b/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeConf.java index 13f870b622..b7e99d2557 100644 --- a/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeConf.java +++ b/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeConf.java @@ -111,7 +111,7 @@ public AzureDataLakeStoragePlugin newPlugin(SabotContext context, String name, P Preconditions.checkNotNull(accountName, "Account name must be set."); Preconditions.checkNotNull(clientId, "Client ID must be set."); Preconditions.checkNotNull(mode, "Authentication mode must be set."); - return new AzureDataLakeStoragePlugin(this, context, name, null, pluginIdProvider); + return new AzureDataLakeStoragePlugin(this, context, name, pluginIdProvider); } @Override diff --git a/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeStoragePlugin.java b/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeStoragePlugin.java index e403cd0f0a..4222b99fa0 100644 --- a/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeStoragePlugin.java +++ b/plugins/s3/src/main/java/com/dremio/plugins/adl/AzureDataLakeStoragePlugin.java @@ -24,7 +24,6 @@ import com.dremio.exec.catalog.conf.Property; import com.dremio.exec.server.SabotContext; import com.dremio.exec.store.dfs.FileSystemPlugin; -import com.dremio.exec.store.dfs.FileSystemWrapper; /** * Storage plugin for Microsoft Azure Data Lake @@ -41,9 +40,9 @@ public class AzureDataLakeStoragePlugin extends FileSystemPlugin idProvider) { - super(config, context, name, fs, idProvider); + super(config, context, name, idProvider); } @Override diff --git a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java index 10d97954b2..5ba8ace484 100644 --- a/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java +++ b/plugins/s3/src/main/java/com/dremio/plugins/s3/store/S3StoragePlugin.java @@ -15,6 +15,7 @@ */ package com.dremio.plugins.s3.store; +import static com.dremio.service.users.SystemUser.SYSTEM_USERNAME; import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY; import static org.apache.hadoop.fs.s3a.Constants.FAST_UPLOAD; import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS; @@ -49,7 +50,6 @@ import com.dremio.exec.store.SchemaConfig; import com.dremio.exec.store.dfs.FileSystemPlugin; import com.dremio.exec.store.dfs.FileSystemWrapper; -import com.dremio.exec.util.ImpersonationUtil; import com.dremio.plugins.s3.store.copy.S3Constants; import com.dremio.plugins.util.ContainerFileSystem.ContainerFailure; import com.dremio.service.namespace.NamespaceKey; @@ -76,7 +76,7 @@ public class S3StoragePlugin extends FileSystemPlugin { public static final String NONE_PROVIDER = "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"; public S3StoragePlugin(S3PluginConfig config, SabotContext context, String name, Provider idProvider) { - super(config, context, name, null, idProvider); + super(config, context, name, idProvider); } @Override @@ -137,7 +137,7 @@ protected List getProperties() { public SourceState getState() { try { ensureDefaultName(); - S3FileSystem fs = getFS(ImpersonationUtil.getProcessUserName()).unwrap(S3FileSystem.class); + S3FileSystem fs = getSystemUserFS().unwrap(S3FileSystem.class); fs.refreshFileSystems(); List failures = fs.getSubFailures(); if(failures.isEmpty()) { @@ -161,7 +161,8 @@ public SourceState getState() { private void ensureDefaultName() throws IOException { String urlSafeName = URLEncoder.encode(getName(), "UTF-8"); getFsConf().set(FileSystem.FS_DEFAULT_NAME_KEY, "dremioS3://" + urlSafeName); - final FileSystemWrapper fs = getFS(ImpersonationUtil.getProcessUserName()); + // we create a new fs wrapper since we are calling initialize on it + final FileSystemWrapper fs = createFS(SYSTEM_USERNAME); // do not use fs.getURI() or fs.getConf() directly as they will produce wrong results fs.initialize(URI.create(getFsConf().get(FileSystem.FS_DEFAULT_NAME_KEY)), getFsConf()); } @@ -191,7 +192,7 @@ public CreateTableEntry createNewTable( final CreateTableEntry entry = super.createNewTable(config, key, writerOptions, storageOptions); - final S3FileSystem fs = getFS(ImpersonationUtil.getProcessUserName()).unwrap(S3FileSystem.class); + final S3FileSystem fs = getSystemUserFS().unwrap(S3FileSystem.class); if (!fs.containerExists(containerName)) { throw UserException.validationError() diff --git a/pom.xml b/pom.xml index 86900f595f..df103e133e 100755 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 pom Dremio - Root diff --git a/protocol/pom.xml b/protocol/pom.xml index b745fe007f..fa99fff33f 100644 --- a/protocol/pom.xml +++ b/protocol/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio diff --git a/provision/common/pom.xml b/provision/common/pom.xml index 21d52ec2de..a2befbb418 100644 --- a/provision/common/pom.xml +++ b/provision/common/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-provision-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.provision diff --git a/provision/pom.xml b/provision/pom.xml index 368eb5cbbd..0c1ec65eb0 100644 --- a/provision/pom.xml +++ b/provision/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-provision-parent diff --git a/provision/shimloader/pom.xml b/provision/shimloader/pom.xml index 039cfabf5a..9a73b31ce0 100644 --- a/provision/shimloader/pom.xml +++ b/provision/shimloader/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-provision-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.provision diff --git a/provision/yarn/pom.xml b/provision/yarn/pom.xml index 5164b06877..685ef38d29 100644 --- a/provision/yarn/pom.xml +++ b/provision/yarn/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-provision-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.provision diff --git a/provision/yarn/twill/pom.xml b/provision/yarn/twill/pom.xml index 636132255f..91f5f5917b 100644 --- a/provision/yarn/twill/pom.xml +++ b/provision/yarn/twill/pom.xml @@ -22,7 +22,7 @@ com.dremio.provision dremio-yarn-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.provision diff --git a/provision/yarn/yarntwill/pom.xml b/provision/yarn/yarntwill/pom.xml index 3d42fc9f13..2379b4a749 100644 --- a/provision/yarn/yarntwill/pom.xml +++ b/provision/yarn/yarntwill/pom.xml @@ -22,7 +22,7 @@ com.dremio.provision dremio-yarn-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.provision diff --git a/sabot/kernel/pom.xml b/sabot/kernel/pom.xml index abadabb2c4..0a6b349125 100644 --- a/sabot/kernel/pom.xml +++ b/sabot/kernel/pom.xml @@ -21,7 +21,7 @@ com.dremio.sabot dremio-sabot-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-sabot-kernel Sabot - Kernel @@ -218,7 +218,7 @@ org.mongodb mongo-java-driver - 3.0.2 + 3.6.4 com.fasterxml.jackson.module diff --git a/sabot/kernel/src/main/java/com/dremio/exec/ExecConstants.java b/sabot/kernel/src/main/java/com/dremio/exec/ExecConstants.java index cfe1309216..ec68b0452d 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/ExecConstants.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/ExecConstants.java @@ -376,4 +376,7 @@ public interface ExecConstants { LongValidator VOTING_SCHEDULE = new PositiveLongValidator("vote.schedule.millis", Long.MAX_VALUE, 0); PositiveLongValidator LAST_SEARCH_REINDEX = new PositiveLongValidator("dac.search.last_reindex", Long.MAX_VALUE, 0); PositiveLongValidator SEARCH_MANAGER_REFRESH_MILLIS = new PositiveLongValidator("dac.search.refresh", Long.MAX_VALUE, TimeUnit.MINUTES.toMillis(1)); + + //this option sets the capacity of an ArrowBuf in SlicedBufferManager from which various buffers may be sliced. + PowerOfTwoLongValidator BUF_MANAGER_CAPACITY = new PowerOfTwoLongValidator("exec.sliced_bufmgr.capacity", 1 << 24, 1 << 16); } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java b/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java index 161bd8b34e..ba22d54fb7 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetManager.java @@ -299,7 +299,7 @@ private DremioTable getTableFromPlugin(NamespaceKey key, DatasetConfig datasetCo final DatasetConfig newDatasetConfig = tableDefinition.getDataset(); final List splits = tableDefinition.getSplits(); NamespaceUtils.copyFromOldConfig(datasetConfig, newDatasetConfig); - if (BatchSchema.fromDataset(newDatasetConfig).getFieldCount() > maxMetadataColumns) { + if (BatchSchema.fromDataset(newDatasetConfig).getTotalFieldCount() > maxMetadataColumns) { throw UserException.validationError() .message(String.format("Using datasets with more than %d columns is currently disabled.", maxMetadataColumns)) diff --git a/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetSaver.java b/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetSaver.java index b69ae69da2..056837f501 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetSaver.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/catalog/DatasetSaver.java @@ -74,7 +74,7 @@ void datasetSave( try { DatasetConfig newConfig = accessor.getDataset(); - if (BatchSchema.fromDataset(newConfig).getFieldCount() > maxMetadataLeafColumns) { + if (BatchSchema.fromDataset(newConfig).getTotalFieldCount() > maxMetadataLeafColumns) { NamespaceUtils.copyFromOldConfig(oldConfig, newConfig); newConfig.setRecordSchema(null); newConfig.setReadDefinition(null); diff --git a/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/HLLAccum.java b/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/HLLAccum.java index d07ce727c3..91f82310ae 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/HLLAccum.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/HLLAccum.java @@ -20,6 +20,8 @@ import org.apache.arrow.memory.BufferManager; import org.apache.arrow.vector.holders.ObjectHolder; +import com.dremio.sabot.exec.context.SlicedBufferManager; + import com.yahoo.memory.Memory; import com.yahoo.memory.WritableMemory; import com.yahoo.sketches.hll.HllSketch; @@ -33,12 +35,11 @@ @SuppressWarnings("deprecation") public final class HLLAccum { - private final ArrowBuf buf; private final HllSketch sketch; private HLLAccum(BufferManager manager, int lgConfigK) { final int size = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, TgtHllType.HLL_8); - this.buf = manager.getManagedBuffer(size); + final ArrowBuf buf = ((SlicedBufferManager) manager).getManagedBufferSliced(size); buf.setZero(0, size); this.sketch = new HllSketch(lgConfigK, TgtHllType.HLL_8, WritableMemory.wrap(buf.nioBuffer(0, size))); } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/UnionAccum.java b/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/UnionAccum.java index 6ac53ee033..6310db3e6d 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/UnionAccum.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/expr/fn/hll/UnionAccum.java @@ -18,6 +18,7 @@ import org.apache.arrow.memory.BufferManager; import org.apache.arrow.vector.holders.ObjectHolder; +import com.dremio.sabot.exec.context.SlicedBufferManager; import com.yahoo.memory.Memory; import com.yahoo.memory.WritableMemory; import com.yahoo.sketches.hll.HllSketch; @@ -32,12 +33,11 @@ */ public class UnionAccum { - private final ArrowBuf buf; private final Union union; private UnionAccum(BufferManager manager, int lgConfigK) { final int size = HllSketch.getMaxUpdatableSerializationBytes(lgConfigK, TgtHllType.HLL_8); - this.buf = manager.getManagedBuffer(size); + final ArrowBuf buf = ((SlicedBufferManager) manager).getManagedBufferSliced(size); buf.setZero(0, size); this.union = new Union(lgConfigK, WritableMemory.wrap(buf.nioBuffer(0, size))); } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemDatasetAccessor.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemDatasetAccessor.java index 77f6fa661e..4c859eb086 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemDatasetAccessor.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemDatasetAccessor.java @@ -167,7 +167,7 @@ protected final DatasetConfig getDatasetInternal( if (oldConfig != null && DatasetHelper.getSchemaBytes(oldConfig) != null) { schema = BatchSchema.fromDataset(oldConfig).merge(newSchema); } - if (schema.getFieldCount() > maxLeafColumns) { + if (schema.getTotalFieldCount() > maxLeafColumns) { throw new ColumnCountTooLargeException( String.format("Using datasets with more than %d columns is currently disabled.", maxLeafColumns)); } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemPlugin.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemPlugin.java index e3cf589d84..f880ca5aa4 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemPlugin.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/FileSystemPlugin.java @@ -133,7 +133,7 @@ public class FileSystemPlugin> implements Storage private final Path basePath; private final Provider idProvider; - private FileSystemWrapper fs; + private FileSystemWrapper systemUserFS; private Configuration fsConf; private FormatPluginOptionExtractor optionExtractor; protected FormatCreator formatCreator; @@ -141,11 +141,10 @@ public class FileSystemPlugin> implements Storage private List dropFileMatchers; private CompressionCodecFactory codecFactory; - public FileSystemPlugin(final C config, final SabotContext context, final String name, FileSystemWrapper fs, Provider idProvider) { + public FileSystemPlugin(final C config, final SabotContext context, final String name, Provider idProvider) { this.name = name; this.config = config; this.idProvider = idProvider; - this.fs = fs; this.context = context; this.fsConf = getNewFsConf(); this.lpPersistance = context.getLpPersistence(); @@ -156,6 +155,10 @@ public C getConfig(){ return config; } + public boolean supportsColocatedReads() { + return true; + } + @Override public Iterable getDatasets(String user, DatasetRetrievalOptions ignored) throws Exception { return Collections.emptyList(); // file system does not know about physical datasets @@ -163,7 +166,7 @@ public Iterable getDatasets(String user, DatasetRetrieval @Override public SourceCapabilities getSourceCapabilities() { - return fs.isPdfs() ? new SourceCapabilities(REQUIRES_HARD_AFFINITY) : SourceCapabilities.NONE; + return systemUserFS.isPdfs() ? new SourceCapabilities(REQUIRES_HARD_AFFINITY) : SourceCapabilities.NONE; } public static final Configuration getNewFsConf() { @@ -178,13 +181,19 @@ public StoragePluginId getId() { return idProvider.get(); } - public FileSystemWrapper getFS(String userName) { - return getFs(userName, null); + /** + * Create a new {@link FileSystemWrapper} for a given user. + * + * @param userName + * @return + */ + public FileSystemWrapper createFS(String userName) { + return createFS(userName, null); } - public FileSystemWrapper getFs(String userName, OperatorStats stats) { + public FileSystemWrapper createFS(String userName, OperatorStats stats) { return ImpersonationUtil.createFileSystem(getUGIForUser(userName), getFsConf(), stats, - getConnectionUniqueProperties()); + getConnectionUniqueProperties()); } public UserGroupInformation getUGIForUser(String userName) { @@ -203,7 +212,7 @@ public Iterable getSubPartitions(List table, List fileStatuses; try { Path fullPath = PathUtils.toFSPath(resolveTableNameToValidPath(table)); - fileStatuses = getFS(schemaConfig.getUserName()).list(fullPath, false); + fileStatuses = createFS(schemaConfig.getUserName()).list(fullPath, false); } catch (IOException e) { throw new PartitionNotFoundException("Error finding partitions for table " + table, e); } @@ -217,10 +226,9 @@ public Class getRulesFactoryClass() { @Override public SourceState getState() { - final FileSystemWrapper fs = getFS(ImpersonationUtil.getProcessUserName()); - if (!fs.isPdfs()) { + if (!systemUserFS.isPdfs()) { try { - fs.listStatus(config.getPath()); + systemUserFS.listStatus(config.getPath()); return SourceState.GOOD; } catch (Exception e) { return SourceState.badState(e); @@ -235,7 +243,7 @@ public ViewTable getView(List tableSchemaPath, SchemaConfig schemaConfig List files = Collections.emptyList(); try { try { - files = DotFileUtil.getDotFiles(getFS(schemaConfig.getUserName()), config.getPath(), tableSchemaPath.get(tableSchemaPath.size() - 1), DotFileType.VIEW); + files = DotFileUtil.getDotFiles(createFS(schemaConfig.getUserName()), config.getPath(), tableSchemaPath.get(tableSchemaPath.size() - 1), DotFileType.VIEW); } catch (AccessControlException e) { if (!schemaConfig.getIgnoreAuthErrors()) { logger.debug(e.getMessage()); @@ -347,7 +355,7 @@ protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, D // TODO: why do we need distinguish between system user and process user? final String userName = config.isImpersonationEnabled() ? SystemUser.SYSTEM_USERNAME : ImpersonationUtil.getProcessUserName(); List parentSchemaPath = new ArrayList<>(fullPath.subList(0, fullPath.size() - 1)); - FileSystemWrapper fs = getFS((user != null) ? user : userName); + FileSystemWrapper fs = createFS((user != null) ? user : userName); FileSelection fileSelection = FileSelection.create(fs, fullPath); String tableName = datasetPath.getName(); @@ -459,11 +467,8 @@ public void start() throws IOException { } // boolean footerNoSeek = contetMutext.getOptionManager().getOption(ExecConstants.PARQUET_FOOTER_NOSEEK); - // NOTE: Add fallback format matcher if given in the configuration. Make sure fileMatchers is an order-preserving list. - if(fs == null) { - this.fs = getFS(SYSTEM_USERNAME); - } + this.systemUserFS = createFS(SYSTEM_USERNAME); dropFileMatchers = matchers.subList(0, matchers.size()); createIfNecessary(); @@ -484,10 +489,10 @@ private void createIfNecessary() throws IOException { try { // no need to exists here as FileSystemWrapper does an exists check and this is a noop if already existing. - fs.mkdirs(config.getPath()); + systemUserFS.mkdirs(config.getPath()); } catch (IOException ex) { try { - if(fs.exists(config.getPath())) { + if(systemUserFS.exists(config.getPath())) { // race creation, ignore. return; } @@ -524,7 +529,7 @@ protected FileSystemCachedEntity fromFileStatus(FileStatus status){ public boolean hasAccessPermission(String user, NamespaceKey key, DatasetConfig datasetConfig) { if (config.isImpersonationEnabled()) { if (datasetConfig.getReadDefinition() != null) { // allow accessing partial datasets - final FileSystemWrapper userFs = getFS(user); + final FileSystemWrapper userFs = createFS(user); final List> permissionCheckTasks = Lists.newArrayList(); permissionCheckTasks.addAll(getUpdateKeyPermissionTasks(datasetConfig, userFs)); @@ -642,8 +647,13 @@ protected Boolean runInner() throws Exception { } } - public FileSystemWrapper getFs() { - return fs; + /** + * Returns a shared {@link FileSystemWrapper} for the system user. + * + * @return + */ + public FileSystemWrapper getSystemUserFS() { + return systemUserFS; } public String getName() { @@ -654,7 +664,7 @@ public String getName() { public boolean containerExists(NamespaceKey key) { final List folderPath = key.getPathComponents(); try { - return getFS(SYSTEM_USERNAME).isDirectory(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath))); + return systemUserFS.isDirectory(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath))); } catch (IOException e) { logger.debug("Failure reading path.", e); return false; @@ -665,7 +675,7 @@ public boolean containerExists(NamespaceKey key) { public boolean datasetExists(NamespaceKey key) { final List filePath = key.getPathComponents(); try { - return getFS(SYSTEM_USERNAME).exists(PathUtils.toFSPath(resolveTableNameToValidPath(filePath))); + return systemUserFS.exists(PathUtils.toFSPath(resolveTableNameToValidPath(filePath))); } catch (IOException e) { logger.debug("Failure reading path.", e); return false; @@ -673,7 +683,7 @@ public boolean datasetExists(NamespaceKey key) { } protected boolean fileExists(String username, List filePath) throws IOException { - return getFS(username).isFile(PathUtils.toFSPath(resolveTableNameToValidPath(filePath))); + return createFS(username).isFile(PathUtils.toFSPath(resolveTableNameToValidPath(filePath))); } @Override @@ -746,7 +756,7 @@ private UpdateStatus checkMultifileStatus(FileUpdateKey fileUpdateKey) { final Path cachedEntityPath = new Path(cachedEntity.getPath()); try { - final Optional optionalStatus = fs.getFileStatusSafe(cachedEntityPath); + final Optional optionalStatus = systemUserFS.getFileStatusSafe(cachedEntityPath); if(!optionalStatus.isPresent()) { // if first entity (root) is missing then table is deleted if (i == 0) { @@ -796,7 +806,7 @@ public boolean createOrUpdateView(NamespaceKey key, View view, SchemaConfig sche } Path viewPath = getViewPath(key.getPathComponents()); - FileSystemWrapper fs = getFS(schemaConfig.getUserName()); + FileSystemWrapper fs = createFS(schemaConfig.getUserName()); boolean replaced = fs.exists(viewPath); final FsPermission viewPerms = new FsPermission(schemaConfig.getOption(ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY).getStringVal()); @@ -814,7 +824,7 @@ public void dropView(SchemaConfig schemaConfig, List tableSchemaPath) th .build(logger); } - getFS(schemaConfig.getUserName()).delete(getViewPath(tableSchemaPath), false); + createFS(schemaConfig.getUserName()).delete(getViewPath(tableSchemaPath), false); } private Path getViewPath(List tableSchemaPath) { @@ -878,7 +888,7 @@ public void dropTable(List tableSchemaPath, SchemaConfig schemaConfig) { .build(logger); } - FileSystemWrapper fs = getFS(schemaConfig.getUserName()); + FileSystemWrapper fs = createFS(schemaConfig.getUserName()); List fullPath = resolveTableNameToValidPath(tableSchemaPath); FileSelection fileSelection; try { @@ -928,7 +938,7 @@ public void dropTable(List tableSchemaPath, SchemaConfig schemaConfig) { */ public SchemaEntity get(List path, String userName) { try { - final FileStatus status = getFS(userName).getFileStatus(PathUtils.toFSPath(resolveTableNameToValidPath(path))); + final FileStatus status = createFS(userName).getFileStatus(PathUtils.toFSPath(resolveTableNameToValidPath(path))); final Set> tableNames = Sets.newHashSet(); final NamespaceService ns = context.getNamespaceService(userName); @@ -951,7 +961,7 @@ public SchemaEntity get(List path, String userName) { public List list(List folderPath, String userName) { try { - final List files = Lists.newArrayList(getFS(userName).listStatus(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath)))); + final List files = Lists.newArrayList(createFS(userName).listStatus(PathUtils.toFSPath(resolveTableNameToValidPath(folderPath)))); final Set> tableNames = Sets.newHashSet(); final NamespaceService ns = context.getNamespaceService(userName); final NamespaceKey folderNSKey = new NamespaceKey(folderPath); @@ -1011,10 +1021,6 @@ public FormatPluginConfig createConfigForTable(String tableName, Map getConnectionUniqueProperties() { return config.getConnectionUniqueProperties(); } @@ -1057,7 +1063,7 @@ public CreateTableEntry createNewTable(SchemaConfig config, NamespaceKey key, Wr // check that there is no directory at the described path. Path path = resolveTablePathToValidPath(tableName); try { - if(fs.exists(path)) { + if(systemUserFS.exists(path)) { throw UserException.validationError().message("Folder already exists at path: %s.", key).build(logger); } } catch (IOException e) { diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSConf.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSConf.java index fd75da16a9..abab8f19e7 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSConf.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSConf.java @@ -118,7 +118,7 @@ public String getShortCircuitSocketPath() { @Override public HDFSStoragePlugin newPlugin(SabotContext context, String name, Provider idProvider) { - return new HDFSStoragePlugin(this, context, name, null, idProvider); + return new HDFSStoragePlugin(this, context, name, idProvider); } } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSStoragePlugin.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSStoragePlugin.java index 7c93c5e1f7..d148759f19 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSStoragePlugin.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/HDFSStoragePlugin.java @@ -38,9 +38,8 @@ public class HDFSStoragePlugin extends FileSystemPlugin { private static final String HDFS_READ_SHORTCIRCUIT_KEY = "dfs.client.read.shortcircuit"; private static final String HDFS_DFS_DOMAIN_SOCKET_PATH_KEY = "dfs.domain.socket.path"; - public HDFSStoragePlugin(HDFSConf config, SabotContext context, String name, FileSystemWrapper fs, - Provider idProvider) { - super(config, context, name, fs, idProvider); + public HDFSStoragePlugin(HDFSConf config, SabotContext context, String name, Provider idProvider) { + super(config, context, name, idProvider); // TODO Auto-generated constructor stub } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/InternalFileConf.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/InternalFileConf.java index 3a71011d1b..0730625e32 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/InternalFileConf.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/InternalFileConf.java @@ -84,7 +84,7 @@ public SchemaMutability getSchemaMutability() { @Override public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) { - return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider); + return new FileSystemPlugin<>(this, context, name, pluginIdProvider); } public InternalFileConf() { diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/MapRConf.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/MapRConf.java index ea570e5d1b..83fa97437d 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/MapRConf.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/MapRConf.java @@ -96,7 +96,7 @@ public SchemaMutability getSchemaMutability() { @Override public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) { - return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider); + return new FileSystemPlugin<>(this, context, name, pluginIdProvider); } } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/NASConf.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/NASConf.java index c4072c101c..362fa173ae 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/NASConf.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/NASConf.java @@ -74,7 +74,7 @@ public SchemaMutability getSchemaMutability() { @Override public FileSystemPlugin newPlugin(SabotContext context, String name, Provider pluginIdProvider) { - return new FileSystemPlugin<>(this, context, name, null, pluginIdProvider); + return new FileSystemPlugin<>(this, context, name, pluginIdProvider); } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyScanOperatorCreator.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyScanOperatorCreator.java index f247e86729..b6c6e348cd 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyScanOperatorCreator.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyScanOperatorCreator.java @@ -89,7 +89,7 @@ public EasyDatasetSplitXAttr getExtended() { public ProducerOperator create(FragmentExecutionContext fragmentExecContext, final OperatorContext context, EasySubScan config) throws ExecutionSetupException { final FileSystemPlugin plugin = fragmentExecContext.getStoragePlugin(config.getPluginId()); - final FileSystemWrapper fs = plugin.getFs(config.getUserName(), context.getStats()); + final FileSystemWrapper fs = plugin.createFS(config.getUserName(), context.getStats()); final FormatPluginConfig formatConfig = PhysicalDatasetUtils.toFormatPlugin(config.getFileConfig(), Collections.emptyList()); final EasyFormatPlugin formatPlugin = (EasyFormatPlugin) plugin.getFormatPlugin(formatConfig); diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyWriter.java b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyWriter.java index 815b7f5fbf..700e511d2f 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyWriter.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/dfs/easy/EasyWriter.java @@ -112,6 +112,6 @@ public int getOperatorType() { @Override @JsonIgnore public boolean isPdfs() { - return plugin.getFs().isPdfs(); + return plugin.getSystemUserFS().isPdfs(); } } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/ExcelUtil.java b/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/ExcelUtil.java index b0f518784b..e297b3bc7d 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/ExcelUtil.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/ExcelUtil.java @@ -35,6 +35,8 @@ public class ExcelUtil { public static final String ROW = "row"; public static final String CELL = "c"; public static final String VALUE = "v"; + public static final String INLINE_STRING = "is"; + public static final String INLINE_STRING_TEXT = "t"; public static final String SST_STRING = "s"; public static final String MERGE_CELLS = "mergeCells"; public static final String MERGE_CELL = "mergeCell"; diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/StAXBasedParser.java b/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/StAXBasedParser.java index f587c7e6ee..a366a9a194 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/StAXBasedParser.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/easy/excel/StAXBasedParser.java @@ -97,6 +97,7 @@ public int compare(MergeCellRegion o1, MergeCellRegion o2) { private MinorType valueTypeFromStyle; private Map styleToTypeCache = Maps.newHashMap(); + private Map indexToLastTypeCache = Maps.newHashMap(); private final OPCPackage pkgInputStream; @@ -202,19 +203,23 @@ private void parseHeaders() throws XMLStreamException { boolean inValue = false; boolean lookupValueInSST = false; + boolean inInlineString = false; while (xmlStreamReader.hasNext()) { xmlStreamReader.next(); final int event = xmlStreamReader.getEventType(); switch (event) { case START_ELEMENT: { final String name = xmlStreamReader.getLocalName(); - if (ExcelUtil.VALUE.equals(name)) { + if (ExcelUtil.VALUE.equals(name) || + (inInlineString && ExcelUtil.INLINE_STRING_TEXT.equals(name))) { inValue = true; } else if (ExcelUtil.CELL.equals(name)) { lookupValueInSST = ExcelUtil.SST_STRING.equals(xmlStreamReader.getAttributeValue(/*namespaceURI=*/null, ExcelUtil.TYPE)); String cellRef = xmlStreamReader.getAttributeValue(/*namespaceURI=*/null, ExcelUtil.CELL_REF); String columnNameFromRef = ExcelUtil.getColumnName(cellRef); columnIndex = CellReference.convertColStringToIndex(columnNameFromRef); + } else if (ExcelUtil.INLINE_STRING.equals(name)) { + inInlineString = true; } break; } @@ -229,9 +234,12 @@ private void parseHeaders() throws XMLStreamException { case END_ELEMENT: { final String name = xmlStreamReader.getLocalName(); - if (ExcelUtil.VALUE.equals(name)) { + if (ExcelUtil.VALUE.equals(name) || + (inInlineString && ExcelUtil.INLINE_STRING_TEXT.equals(name))) { inValue = false; lookupValueInSST = false; + } else if (ExcelUtil.INLINE_STRING.equals(name)) { + inInlineString = false; } else if (ExcelUtil.ROW.equals(name)) { return; } @@ -292,16 +300,20 @@ public State parseNextRecord() throws XMLStreamException { writer.start(); try { boolean inValue = false; + boolean inInlineString = false; while (xmlStreamReader.hasNext()) { xmlStreamReader.next(); final int event = xmlStreamReader.getEventType(); switch (event) { case START_ELEMENT: { final String name = xmlStreamReader.getLocalName(); - if (ExcelUtil.VALUE.equals(name)) { + if (ExcelUtil.VALUE.equals(name) || + (inInlineString && ExcelUtil.INLINE_STRING_TEXT.equals(name))) { inValue = true; } else if (ExcelUtil.CELL.equals(name)) { handleCellStart(); + } else if (ExcelUtil.INLINE_STRING.equals(name)) { + inInlineString = true; } break; } @@ -315,7 +327,17 @@ public State parseNextRecord() throws XMLStreamException { case END_ELEMENT: { final String name = xmlStreamReader.getLocalName(); - if (ExcelUtil.VALUE.equals(name)) { + if (ExcelUtil.VALUE.equals(name) || + (inInlineString && ExcelUtil.INLINE_STRING_TEXT.equals(name))) { + currentColumnIndex = -1; + lookupNextValueInSST = false; + valueTypeFromAttribute = null; + valueTypeFromStyle = null; + inValue = false; + } else if (ExcelUtil.INLINE_STRING.equals(name)) { + inInlineString = false; + } else if (-1 != currentColumnIndex && ExcelUtil.CELL.equals(name)) { + ensureNull(); currentColumnIndex = -1; lookupNextValueInSST = false; valueTypeFromAttribute = null; @@ -405,16 +427,17 @@ private void handleValue(String value) { final String finalColumnName = columnNameHandler.getColumnName(currentColumnIndex); - final boolean isProjected = (columnsToProject == null) || (columnsToProject.contains(finalColumnName)); + final boolean isProjected = (columnsToProject == null) || columnsToProject.contains(finalColumnName); final MergeCellRegion mergeCellRegion = mergeCells != null ? mergeCells.get(currentCellRef) : null; switch (valueTypeFromAttribute) { case BIT: final int toWrite = "1".equalsIgnoreCase(value) ? 1 : 0; + indexToLastTypeCache.put(currentColumnIndex, valueTypeFromAttribute); if(isProjected) { - writer.bit(finalColumnName).writeBit(toWrite); - } + writer.bit(finalColumnName).writeBit(toWrite); + } if (mergeCellRegion != null) { mergeCellRegion.setValue(MinorType.BIT, toWrite); } @@ -423,6 +446,7 @@ private void handleValue(String value) { case FLOAT8: double dValue = Double.valueOf(value); if (valueTypeFromStyle == MinorType.TIMESTAMP && DateUtil.isValidExcelDate(dValue)) { + indexToLastTypeCache.put(currentColumnIndex, MinorType.TIMESTAMP); final long dateMillis = DateUtil.getJavaDate(dValue, false, LocaleUtil.TIMEZONE_UTC).getTime(); if(isProjected){ writer.timeStampMilli(finalColumnName).writeTimeStampMilli(dateMillis); @@ -431,6 +455,7 @@ private void handleValue(String value) { mergeCellRegion.setValue(MinorType.TIMESTAMP, dateMillis); } } else { + indexToLastTypeCache.put(currentColumnIndex, valueTypeFromAttribute); if(isProjected) { writer.float8(finalColumnName).writeFloat8(dValue); } @@ -441,6 +466,7 @@ private void handleValue(String value) { break; case VARCHAR: + indexToLastTypeCache.put(currentColumnIndex, valueTypeFromAttribute); final byte[] b = value.getBytes(Charsets.UTF_8); managedBuf = managedBuf.reallocIfNeeded(b.length); managedBuf.setBytes(0, b); @@ -454,8 +480,51 @@ private void handleValue(String value) { default: throw UserException.unsupportedError() - .message("Unknown type received %s", valueTypeFromAttribute) - .build(logger); + .message("Unknown type received %s", valueTypeFromAttribute) + .build(logger); + } + } + } + + /** + * Ensure a null value is set for the writer by retrieving the writer for the column but not writing a value. + */ + private void ensureNull() { + assert currentColumnIndex != -1 : "Invalid currentColumnIndex"; + + final String finalColumnName = columnNameHandler.getColumnName(currentColumnIndex); + + if ((columnsToProject == null) || columnsToProject.contains(finalColumnName)) { + MinorType type = indexToLastTypeCache.get(currentColumnIndex); + if (type == null) { + if (valueTypeFromAttribute == MinorType.FLOAT8 && valueTypeFromStyle == MinorType.TIMESTAMP) { + type = MinorType.TIMESTAMP; + } else { + type = valueTypeFromAttribute; + } + } + + switch (type) { + case BIT: + writer.bit(finalColumnName); + break; + + case FLOAT8: + writer.float8(finalColumnName); + break; + + case TIMESTAMP: + writer.timeMilli(finalColumnName); + break; + + case VARCHAR: + writer.varChar(finalColumnName); + break; + + default: + throw UserException.unsupportedError() + .message("Unknown type received %s", valueTypeFromAttribute) + .build(logger); } } } @@ -482,20 +551,22 @@ private String resolveValue(String value, boolean lookupNextValueInSST) { * current row onwards. */ private void handleMergedCells() { - Iterator iter = mergeCellsSortedByRowEnd.iterator(); - while(iter.hasNext()) { - final MergeCellRegion mcr = iter.next(); - if (mcr.hasValue() && mcr.containsRow(runningRecordCount)) { - for(int colIndex = mcr.colStart; colIndex <= mcr.colEnd; colIndex++) { - String colName = columnNameHandler.getColumnName(colIndex); - if(columnsToProject == null || columnsToProject.contains(colName)) { - mcr.write(managedBuf, writer, colName); + if (mergeCellsSortedByRowEnd != null) { + Iterator iter = mergeCellsSortedByRowEnd.iterator(); + while (iter.hasNext()) { + final MergeCellRegion mcr = iter.next(); + if (mcr.hasValue() && mcr.containsRow(runningRecordCount)) { + for (int colIndex = mcr.colStart; colIndex <= mcr.colEnd; colIndex++) { + String colName = columnNameHandler.getColumnName(colIndex); + if (columnsToProject == null || columnsToProject.contains(colName)) { + mcr.write(managedBuf, writer, colName); + } } } - } - if (mcr.rowEnd < runningRecordCount) { - iter.remove(); + if (mcr.rowEnd < runningRecordCount) { + iter.remove(); + } } } } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetFormatDatasetAccessor.java b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetFormatDatasetAccessor.java index 099e88526b..a73ca8d737 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetFormatDatasetAccessor.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetFormatDatasetAccessor.java @@ -175,7 +175,7 @@ public BatchSchema getBatchSchema(final FileSelection selection, final FileSyste mutator.getContainer().buildSchema(BatchSchema.SelectionVectorMode.NONE); final BatchSchema schema = mutator.getContainer().getSchema(); - if (schema.getFieldCount() > maxLeafColumns) { + if (schema.getTotalFieldCount() > maxLeafColumns) { throw new ColumnCountTooLargeException( String.format("Using datasets with more than %d columns is currently disabled.", maxLeafColumns)); } diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetOperatorCreator.java b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetOperatorCreator.java index 42ec19d5e8..b1ba248fad 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetOperatorCreator.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetOperatorCreator.java @@ -60,7 +60,7 @@ public class ParquetOperatorCreator implements Creator { @Override public ProducerOperator create(FragmentExecutionContext fragmentExecContext, final OperatorContext context, final ParquetSubScan config) throws ExecutionSetupException { final FileSystemPlugin plugin = fragmentExecContext.getStoragePlugin(config.getPluginId()); - final FileSystemWrapper fs = plugin.getFs(config.getUserName(), context.getStats()); + final FileSystemWrapper fs = plugin.createFS(config.getUserName(), context.getStats()); final Stopwatch watch = Stopwatch.createStarted(); diff --git a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetWriter.java b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetWriter.java index 1c7879a63c..2255cea18e 100644 --- a/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetWriter.java +++ b/sabot/kernel/src/main/java/com/dremio/exec/store/parquet/ParquetWriter.java @@ -108,7 +108,7 @@ public int getOperatorType() { @Override @JsonIgnore public boolean isPdfs() { - return plugin.getFs().isPdfs(); + return plugin.getSystemUserFS().isPdfs(); } @JsonIgnore diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/BufferManagerImpl.java b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/BufferManagerImpl.java index 924271dbeb..dde05da409 100644 --- a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/BufferManagerImpl.java +++ b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/BufferManagerImpl.java @@ -17,21 +17,41 @@ import org.apache.arrow.memory.BufferAllocator; -import org.apache.arrow.memory.BufferManager; import com.carrotsearch.hppc.LongObjectHashMap; import com.carrotsearch.hppc.predicates.LongObjectPredicate; import io.netty.buffer.ArrowBuf; -public class BufferManagerImpl implements BufferManager { +public class BufferManagerImpl implements SlicedBufferManager { private LongObjectHashMap managedBuffers = new LongObjectHashMap<>(); private final BufferAllocator allocator; + /** + * This class keeps a pre-allocated large buffer & serve any subsequent allocation + * requests via slicing from this large buffer. + * This helps in reducing the heap allocation overhead as sliced ArrowBuf has smaller footprint. + * Also, the allocation of large buffer may be optimized by doing power-of-2 allocations. + * + * largeBufCapacity: the capacity of the largeBuf + * largeBufUsed tracks: how much memory has been sliced away from the largeBuf + */ + private int largeBufCapacity; + private int largeBufUsed; + private ArrowBuf largeBuf; + + public BufferManagerImpl(BufferAllocator allocator) { this.allocator = allocator; } + public BufferManagerImpl(BufferAllocator allocator, int largeBufCapacity) { + this.allocator = allocator; + this.largeBufCapacity = largeBufCapacity; + this.largeBuf = null; + this.largeBufUsed = 0; + } + @Override public void close() { managedBuffers.forEach(new LongObjectPredicate() { @@ -61,4 +81,23 @@ public ArrowBuf getManagedBuffer(int size) { managedBuffers.put(newBuf.memoryAddress(), newBuf); return newBuf; } + + public ArrowBuf getManagedBufferSliced(int size) { + + if (size >= largeBufCapacity) { + return getManagedBuffer(size); + } + + final int availableSpace = (largeBufCapacity - largeBufUsed); + + if (size > availableSpace || largeBuf == null) { + largeBuf = allocator.buffer(largeBufCapacity, this); + managedBuffers.put(largeBuf.memoryAddress(), largeBuf); + largeBufUsed = 0; + } + + final ArrowBuf buf = largeBuf.slice(largeBufUsed, size); + largeBufUsed += size; + return buf; + } } diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/OperatorContextImpl.java b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/OperatorContextImpl.java index b64a1215ea..8b80c34897 100644 --- a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/OperatorContextImpl.java +++ b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/OperatorContextImpl.java @@ -25,6 +25,7 @@ import com.dremio.common.AutoCloseables; import com.dremio.common.config.SabotConfig; +import com.dremio.exec.ExecConstants; import com.dremio.exec.compile.CodeCompiler; import com.dremio.exec.expr.ClassProducer; import com.dremio.exec.expr.ClassProducerImpl; @@ -94,7 +95,12 @@ public OperatorContextImpl( this.allocator = allocator; this.fragmentOutputAllocator = fragmentOutputAllocator; this.popConfig = popConfig; - this.manager = new BufferManagerImpl(allocator); + + //some unit test cases pass null optionManager + final int bufCapacity = (optionManager != null) ? + (int)optionManager.getOption(ExecConstants.BUF_MANAGER_CAPACITY) : (1 << 16); + this.manager = new BufferManagerImpl(allocator, bufCapacity); + this.stats = stats; this.executionControls = executionControls; this.executor = executor; diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/SlicedBufferManager.java b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/SlicedBufferManager.java new file mode 100644 index 0000000000..a2a149f1c9 --- /dev/null +++ b/sabot/kernel/src/main/java/com/dremio/sabot/exec/context/SlicedBufferManager.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2017-2018 Dremio Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.dremio.sabot.exec.context; + +import org.apache.arrow.memory.BufferManager; + +import io.netty.buffer.ArrowBuf; + + +public interface SlicedBufferManager extends BufferManager { + + ArrowBuf getManagedBufferSliced(int size); + +} diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/hash/HashAggOperator.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/hash/HashAggOperator.java index 6e0339b04e..2c3ea68496 100644 --- a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/hash/HashAggOperator.java +++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/hash/HashAggOperator.java @@ -83,6 +83,9 @@ public class HashAggOperator implements SingleInputOperator { public static final TypeValidators.PositiveLongValidator HASHAGG_MINMAX_CARDINALITY_LIMIT = new TypeValidators.PositiveLongValidator("exec.operator.aggregate.minmax_cardinality_limit", Long.MAX_VALUE, 10000); + //this option sets the capacity of an ArrowBuf in BufferManager from which various buffers may be sliced. + public static final TypeValidators.PowerOfTwoLongValidator BUF_MANAGER_CAPACITY = new TypeValidators.PowerOfTwoLongValidator("exec.operator.aggregate.bufmgr.capacity", 1 << 24, 1 << 16); + private static final ControlsInjector injector = ControlsInjectorFactory.getInjector(HashAggOperator.class); @VisibleForTesting diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggDebug.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggDebug.java index c8f5c23105..b8c9175662 100644 --- a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggDebug.java +++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggDebug.java @@ -168,6 +168,8 @@ void setInfoAfterInit(final long totalPreallocatedMemory, this.aggSchema = (detailedEventTracing ? outgoing.toString() : "enable tracing to record schema"); } + void setMaxVarBlockLength(final int maxVarLen) { this.maxVarBlockLength = maxVarLen; } + void recordOOMEvent(final int iterations, final int ooms, final long currentAllocatedMemory, diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggOperator.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggOperator.java index 5a63026338..b7d280aaa1 100644 --- a/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggOperator.java +++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/aggregate/vectorized/VectorizedHashAggOperator.java @@ -472,6 +472,7 @@ private void setLocalInfoForHashTable() { * the info we need. */ this.maxVariableBlockLength = hashAggPartitions[0].hashTable.getVariableBlockMaxLength(); + debug.setMaxVarBlockLength(this.maxVariableBlockLength); //for debugging purpose this.bitsInChunk = hashAggPartitions[0].hashTable.getBitsInChunk(); this.chunkOffsetMask = hashAggPartitions[0].hashTable.getChunkOffsetMask(); /* hashtable should not have changed the configured value of VECTORIZED_HASHAGG_MAXBATCHSIZE */ @@ -790,11 +791,11 @@ private void consumeDataHelper(final int records) throws Exception { recordsPivoted = BoundedPivots.pivot(pivot, recordsConsumed, stepSize, fixedBlockVector, variableBlockVector); pivotWatch.stop(); + /* STEP 2: then we hash partition the dataset and add pivoted data to multiple hash tables */ long partitionsUsed = insertIntoPartitions(records, recordsPivoted, keyFixedVectorAddr, keyVarVectorAddr, recordsConsumed, 0, 0, false, 0); - if (internalStateMachine == InternalState.SPILL_NEXT_BATCH) { /* insertion for this set of pivoted records failed in between. * we will resume later from the failure point for these set of records and any diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/common/ht2/BoundedPivots.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/common/ht2/BoundedPivots.java index aa0bbfd7fd..1407e1d391 100644 --- a/sabot/kernel/src/main/java/com/dremio/sabot/op/common/ht2/BoundedPivots.java +++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/common/ht2/BoundedPivots.java @@ -143,6 +143,11 @@ private static int pivotVariableLengths( outputRecordIdx++; } + if (outputRecordIdx == 0) { + throw new StringIndexOutOfBoundsException("Not enough space to pivot single record. Allocated capacity for pivot: " + + Long.toString(targetVariable.getMaxMemoryAddress() - targetVariable.getMemoryAddress()) + " bytes."); + } + return outputRecordIdx; } diff --git a/sabot/kernel/src/main/java/com/dremio/sabot/op/writer/WriterCommitterOperator.java b/sabot/kernel/src/main/java/com/dremio/sabot/op/writer/WriterCommitterOperator.java index 4ce0181c3f..d1867cc5bd 100644 --- a/sabot/kernel/src/main/java/com/dremio/sabot/op/writer/WriterCommitterOperator.java +++ b/sabot/kernel/src/main/java/com/dremio/sabot/op/writer/WriterCommitterOperator.java @@ -70,7 +70,7 @@ public VectorAccessible setup(VectorAccessible accessible) throws Exception { throw new IllegalStateException(String.format("Incoming record writer schema doesn't match intended. Expected %s, received %s", RecordWriter.SCHEMA, schema)); } - fs = config.getPlugin().getFS(config.getUserName()); + fs = config.getPlugin().createFS(config.getUserName()); // replacement expression. LogicalExpression replacement; diff --git a/sabot/kernel/src/test/java/com/dremio/exec/store/dfs/TestHDFSStoragePlugin.java b/sabot/kernel/src/test/java/com/dremio/exec/store/dfs/TestHDFSStoragePlugin.java index c2cf3f4b19..b70c0856d4 100644 --- a/sabot/kernel/src/test/java/com/dremio/exec/store/dfs/TestHDFSStoragePlugin.java +++ b/sabot/kernel/src/test/java/com/dremio/exec/store/dfs/TestHDFSStoragePlugin.java @@ -49,7 +49,7 @@ public void testHdfsConfApplied() throws Exception { when(context.getClasspathScan()).thenReturn(DremioTest.CLASSPATH_SCAN_RESULT); Provider idProvider = () -> { return new StoragePluginId(null, conf, null); }; - try(HDFSStoragePlugin fileSystemPlugin = new HDFSStoragePlugin(conf, context, "test-plugin", null, idProvider)) { + try(HDFSStoragePlugin fileSystemPlugin = new HDFSStoragePlugin(conf, context, "test-plugin", idProvider)) { fileSystemPlugin.start(); final Configuration fsConf = fileSystemPlugin.getFsConf(); diff --git a/sabot/kernel/src/test/java/com/dremio/exec/store/excel/TestXlsxExcelFormatPlugin.java b/sabot/kernel/src/test/java/com/dremio/exec/store/excel/TestXlsxExcelFormatPlugin.java index 6a7ecaec1a..5b71d3fe1b 100644 --- a/sabot/kernel/src/test/java/com/dremio/exec/store/excel/TestXlsxExcelFormatPlugin.java +++ b/sabot/kernel/src/test/java/com/dremio/exec/store/excel/TestXlsxExcelFormatPlugin.java @@ -54,9 +54,18 @@ public void testEmpty2Xls() throws Exception { final String filePath = getExcelDir() + "empty2.xlsx"; final String query = String.format("SELECT * FROM TABLE(dfs.\"%s\" (type => 'excel', extractHeader => true, hasMergedCells => true, xls => false))", filePath); - testAndExpectUserException(query, ErrorType.DATA_READ, "Selected table has no columns."); + // There are actual columns in this table, but they contain no value. Hence, show the columns with nulls. + test(query); } + @Test + public void testInlineString() throws Exception { + final String filePath = getExcelDir() + "InlineString.xlsx"; + final String query = String.format("SELECT * FROM TABLE(dfs.\"%s\" (type => 'excel', extractHeader => true, hasMergedCells => false, xls => false))", filePath); + + // This will fail if the inline string column is ignored, as no columns will be detected. + test(query); + } @Test public void testHeaderOnly() throws Exception { diff --git a/sabot/kernel/src/test/java/com/dremio/sabot/CustomHashAggDataGenerator.java b/sabot/kernel/src/test/java/com/dremio/sabot/CustomHashAggDataGenerator.java index 9ff06940d5..2ff5cd556c 100644 --- a/sabot/kernel/src/test/java/com/dremio/sabot/CustomHashAggDataGenerator.java +++ b/sabot/kernel/src/test/java/com/dremio/sabot/CustomHashAggDataGenerator.java @@ -48,17 +48,17 @@ public class CustomHashAggDataGenerator implements Generator { /* fixed width dimensions */ - private final static Field FIXKEY1 = CompleteType.INT.toField("FIXKEY1"); - private final static Field FIXKEY2 = CompleteType.BIGINT.toField("FIXKEY2"); + private static Field FIXKEY1 = CompleteType.INT.toField("FIXKEY1"); + private static Field FIXKEY2 = CompleteType.BIGINT.toField("FIXKEY2"); /* variable width dimensions */ - private final static Field VARKEY1 = CompleteType.VARCHAR.toField("VARKEY1"); + private static Field VARKEY1 = CompleteType.VARCHAR.toField("VARKEY1"); /* Measures */ - private final static Field MEASURE1 = CompleteType.INT.toField("MEASURE1"); - private final static Field MEASURE2 = CompleteType.BIGINT.toField("MEASURE2"); - private final static Field MEASURE3 = CompleteType.FLOAT.toField("MEASURE3"); - private final static Field MEASURE4 = CompleteType.DOUBLE.toField("MEASURE4"); + private static Field MEASURE1 = CompleteType.INT.toField("MEASURE1"); + private static Field MEASURE2 = CompleteType.BIGINT.toField("MEASURE2"); + private static Field MEASURE3 = CompleteType.FLOAT.toField("MEASURE3"); + private static Field MEASURE4 = CompleteType.DOUBLE.toField("MEASURE4"); /* input table */ private Integer[] fixKeyValues1; @@ -69,24 +69,24 @@ public class CustomHashAggDataGenerator implements Generator { private Float[] measure3; private Double[] measure4; - private final VectorContainer container; + private VectorContainer container; /* fixed width key columns */ - private final IntVector fixkey1; - private final BigIntVector fixkey2; + private IntVector fixkey1; + private BigIntVector fixkey2; /* variable width key columns */ - private final VarCharVector varkey1; + private VarCharVector varkey1; /* measure columns */ - private final IntVector m1; - private final BigIntVector m2; - private final Float4Vector m3; - private final Float8Vector m4; + private IntVector m1; + private BigIntVector m2; + private Float4Vector m3; + private Float8Vector m4; private int position; private int batches; - private final boolean largeVarChars; + private boolean largeVarChars; private static final int GROUP_REPEAT_PER_BATCH = 5; /* simplify data generation by using a fixed internal batch size */ @@ -104,13 +104,36 @@ public class CustomHashAggDataGenerator implements Generator { private static final String VARCHAR_BASEVALUE = "Dremio-3.0"; private static final int GROUP_INTERVAL_PER_BATCH = 20; + private int numRows; private final HashMap aggregatedResults = new HashMap<>(); + private int minLargeVarCharLen = 0; + + private void InternalInit(int numRows, BufferAllocator allocator, final boolean largeVarChars) + { + this.numRows = numRows; + this.batches = numRows/BATCH_SIZE; + this.largeVarChars = largeVarChars; + createBigSchemaAndInputContainer(allocator); + } + public CustomHashAggDataGenerator(int numRows, BufferAllocator allocator, final boolean largeVarChars) { + Preconditions.checkState(numRows > 0 && numRows%BATCH_SIZE == 0, + "ERROR: total number of rows should be greater than 0"); + InternalInit(numRows, allocator, largeVarChars); + } + + public CustomHashAggDataGenerator(int numRows, BufferAllocator allocator, + final int minVarCharLen /*minimum length of long varchar*/) { Preconditions.checkState(numRows > 0 && numRows%BATCH_SIZE == 0, "ERROR: total number of rows should be greater than 0"); + this.minLargeVarCharLen = minVarCharLen; + InternalInit(numRows, allocator, true); + } + + private void createBigSchemaAndInputContainer(final BufferAllocator allocator) { BatchSchema schema = BatchSchema.newBuilder() .addField(FIXKEY1) .addField(FIXKEY2) @@ -199,7 +222,11 @@ private void buildInputDataset(int numRows) { /* key columns */ fixKeyValues1[0] = INT_BASEVALUE; fixKeyValues2[0] = BIGINT_BASEVALUE; - varKeyValues1[0] = VARCHAR_BASEVALUE; + if (largeVarChars && minLargeVarCharLen != 0) { + varKeyValues1[0] = RandomStringUtils.randomAlphabetic(minLargeVarCharLen); + } else { + varKeyValues1[0] = VARCHAR_BASEVALUE; + } /* measure columns */ measure1[0] = INT_BASEVALUE; diff --git a/sabot/kernel/src/test/java/com/dremio/sabot/aggregate/hash/TestSpillingHashAgg.java b/sabot/kernel/src/test/java/com/dremio/sabot/aggregate/hash/TestSpillingHashAgg.java index 5b157b26d8..e958587d19 100644 --- a/sabot/kernel/src/test/java/com/dremio/sabot/aggregate/hash/TestSpillingHashAgg.java +++ b/sabot/kernel/src/test/java/com/dremio/sabot/aggregate/hash/TestSpillingHashAgg.java @@ -21,6 +21,7 @@ import java.util.Arrays; import java.util.concurrent.TimeUnit; +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; @@ -82,6 +83,92 @@ public void testNoSpill() throws Exception { } } + /** + * Tests with varchar key of length > 32k + * @throws Exception + */ + @Test + public void testVeryLargeVarcharKey() throws Exception { + final HashAggregate agg = getHashAggregate(); + agg.setInitialAllocation(1_000_000); + agg.setMaxAllocation(12_000_000); + + boolean exceptionThrown = false; + + + final int shortLen = (30 * 1024); + final int largeLen = (32 * 1024); + + //30k size must not fail, any subsequent 32k inserts would fail + boolean shortLenSucess = false; + try (AutoCloseable useSpillingAgg = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_USE_SPILLING_OPERATOR, true)) { + try (CustomHashAggDataGenerator generator = new CustomHashAggDataGenerator(1000, getTestAllocator(), shortLen)) { + try (CustomHashAggDataGenerator generator1 = new CustomHashAggDataGenerator(1000, getTestAllocator(), largeLen)) { + Fixtures.Table table = generator.getExpectedGroupsAndAggregations(); + validateSingle(agg, VectorizedHashAggOperator.class, generator, table, 1000); + + //30k key size should have worked. must reach here! + Assert.assertEquals(0, 0); + + shortLenSucess = true; + //add some 32k keys + Fixtures.Table table1 = generator1.getExpectedGroupsAndAggregations(); + validateSingle(agg, VectorizedHashAggOperator.class, generator1, table1, 1000); + + //must not reach here + Assert.assertEquals(0, 1); + } + } + } catch (StringIndexOutOfBoundsException userExp) { + exceptionThrown = true; + Assert.assertEquals(true, shortLenSucess); + } finally { + Assert.assertEquals(true, exceptionThrown); + } + + //fails to pivot with 32k key size + try (AutoCloseable useSpillingAgg = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_USE_SPILLING_OPERATOR, true)) { + try (CustomHashAggDataGenerator generator = new CustomHashAggDataGenerator(1000, getTestAllocator(), largeLen)) { + Fixtures.Table table = generator.getExpectedGroupsAndAggregations(); + validateSingle(agg, VectorizedHashAggOperator.class, generator, table, 1000); + + //must not reach here + Assert.assertEquals(0, 1); + } + } catch (StringIndexOutOfBoundsException userExp) { + exceptionThrown = true; + } finally { + Assert.assertEquals(true, exceptionThrown); + } + + //passes 32k key size with increased batch of 2k + exceptionThrown = false; + try (AutoCloseable useSpillingAgg = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_USE_SPILLING_OPERATOR, true)) { + try (CustomHashAggDataGenerator generator = new CustomHashAggDataGenerator(1000, getTestAllocator(), largeLen); + AutoCloseable options = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_BATCHSIZE, 2048)) { + Fixtures.Table table = generator.getExpectedGroupsAndAggregations(); + validateSingle(agg, VectorizedHashAggOperator.class, generator, table, 1000); + } + } catch (Exception e) { + exceptionThrown = true; + } finally { + Assert.assertEquals(false, exceptionThrown); + } + + //should fail with 1MB key size + try (AutoCloseable useSpillingAgg = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_USE_SPILLING_OPERATOR, true)) { + try (CustomHashAggDataGenerator generator = new CustomHashAggDataGenerator(1000, getTestAllocator(), (1024 * 1024)); + AutoCloseable options = with(VectorizedHashAggOperator.VECTORIZED_HASHAGG_BATCHSIZE, 4096)) { + Fixtures.Table table = generator.getExpectedGroupsAndAggregations(); + validateSingle(agg, VectorizedHashAggOperator.class, generator, table, 1000); + } + } catch (StringIndexOutOfBoundsException userExp) { + exceptionThrown = true; + } finally { + Assert.assertEquals(true, exceptionThrown); + } + } + /** * Test failure during operator setup when provided memory constraints * are lower than the memory required for preallocating data structures. diff --git a/sabot/kernel/src/test/resources/excel/InlineString.xlsx b/sabot/kernel/src/test/resources/excel/InlineString.xlsx new file mode 100644 index 0000000000..cd1683f344 Binary files /dev/null and b/sabot/kernel/src/test/resources/excel/InlineString.xlsx differ diff --git a/sabot/logical/pom.xml b/sabot/logical/pom.xml index 6eef9158a9..4cb5e9e8e9 100644 --- a/sabot/logical/pom.xml +++ b/sabot/logical/pom.xml @@ -22,7 +22,7 @@ com.dremio.sabot dremio-sabot-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-sabot-logical diff --git a/sabot/pom.xml b/sabot/pom.xml index f9f590d623..cdfa1c7648 100644 --- a/sabot/pom.xml +++ b/sabot/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.sabot diff --git a/services/accelerator/pom.xml b/services/accelerator/pom.xml index 022d2db5e0..e3ba6f4228 100644 --- a/services/accelerator/pom.xml +++ b/services/accelerator/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-accelerator diff --git a/services/accelerator/src/main/java/com/dremio/service/reflection/materialization/AccelerationStoragePlugin.java b/services/accelerator/src/main/java/com/dremio/service/reflection/materialization/AccelerationStoragePlugin.java index 74275117cb..e762cddd75 100644 --- a/services/accelerator/src/main/java/com/dremio/service/reflection/materialization/AccelerationStoragePlugin.java +++ b/services/accelerator/src/main/java/com/dremio/service/reflection/materialization/AccelerationStoragePlugin.java @@ -69,7 +69,7 @@ public class AccelerationStoragePlugin extends FileSystemPlugin idProvider) { - super(config, context, name, null, idProvider); + super(config, context, name, idProvider); } @Override @@ -166,7 +166,7 @@ public SourceTableDefinition getDataset( @Override public Iterable apply(Refresh input) { try { - FileSelection selection = FileSelection.create(getFs(), resolveTablePathToValidPath(input.getPath())); + FileSelection selection = FileSelection.create(getSystemUserFS(), resolveTablePathToValidPath(input.getPath())); if(selection != null) { return selection.minusDirectories().getFileStatuses(); } @@ -177,7 +177,7 @@ public Iterable apply(Refresh input) { }}).toList(); FileSelection selection = FileSelection.createFromExpanded(allStatus, selectionRoot); - return new ParquetFormatDatasetAccessor(oldConfig, getFs(), selection, this, datasetPath, EMPTY, formatPlugin, + return new ParquetFormatDatasetAccessor(oldConfig, getSystemUserFS(), selection, this, datasetPath, EMPTY, formatPlugin, options.maxMetadataLeafColumns()); } diff --git a/services/base-rpc/pom.xml b/services/base-rpc/pom.xml index 3c4715ba7b..5b29f09d6d 100644 --- a/services/base-rpc/pom.xml +++ b/services/base-rpc/pom.xml @@ -21,7 +21,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-base-rpc Services - Base RPC diff --git a/services/configuration/pom.xml b/services/configuration/pom.xml index 0416bea45c..8883a55457 100644 --- a/services/configuration/pom.xml +++ b/services/configuration/pom.xml @@ -22,7 +22,7 @@ dremio-services-parent com.dremio.services - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 4.0.0 dremio-services-configuration diff --git a/services/coordinator/pom.xml b/services/coordinator/pom.xml index f902b90289..63c05255b8 100644 --- a/services/coordinator/pom.xml +++ b/services/coordinator/pom.xml @@ -21,7 +21,7 @@ dremio-services-parent com.dremio.services - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-coordinator Services - Coordinator diff --git a/services/datastore/pom.xml b/services/datastore/pom.xml index 906809ccf2..39fb53fe5d 100644 --- a/services/datastore/pom.xml +++ b/services/datastore/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-datastore Services - Datastore diff --git a/services/fabric-rpc/pom.xml b/services/fabric-rpc/pom.xml index 4b96f1d666..2cb24af383 100644 --- a/services/fabric-rpc/pom.xml +++ b/services/fabric-rpc/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-fabric-rpc Services - Fabric RPC diff --git a/services/jobs/pom.xml b/services/jobs/pom.xml index 98011533ee..86a4143884 100644 --- a/services/jobs/pom.xml +++ b/services/jobs/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-jobs Services - Jobs diff --git a/services/jobs/src/main/java/com/dremio/service/jobs/JobResultsStore.java b/services/jobs/src/main/java/com/dremio/service/jobs/JobResultsStore.java index e5b821bc46..f020bbf6bb 100644 --- a/services/jobs/src/main/java/com/dremio/service/jobs/JobResultsStore.java +++ b/services/jobs/src/main/java/com/dremio/service/jobs/JobResultsStore.java @@ -36,7 +36,6 @@ import com.dremio.datastore.IndexedStore; import com.dremio.exec.store.dfs.FileSystemPlugin; import com.dremio.exec.store.easy.arrow.ArrowFileMetadata; -import com.dremio.exec.util.ImpersonationUtil; import com.dremio.service.Service; import com.dremio.service.job.proto.JobAttempt; import com.dremio.service.job.proto.JobId; @@ -75,7 +74,7 @@ public class JobResultsStore implements Service { public JobResultsStore(final FileSystemPlugin plugin, final IndexedStore store, final BufferAllocator allocator) throws IOException { this.storageName = plugin.getName(); - this.dfs = plugin.getFS(ImpersonationUtil.getProcessUserName()); + this.dfs = plugin.getSystemUserFS(); this.jobStoreLocation = plugin.getConfig().getPath(); this.dfs.mkdirs(jobStoreLocation); this.store = store; diff --git a/services/metrics/pom.xml b/services/metrics/pom.xml index 6207f33ea8..b8e78835a2 100644 --- a/services/metrics/pom.xml +++ b/services/metrics/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-metrics Services - Metrics diff --git a/services/namespace/pom.xml b/services/namespace/pom.xml index 9574eae105..3074249303 100644 --- a/services/namespace/pom.xml +++ b/services/namespace/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-namespace Services - Namespace diff --git a/services/options/pom.xml b/services/options/pom.xml index 3bdc714084..509c7fa9ea 100644 --- a/services/options/pom.xml +++ b/services/options/pom.xml @@ -23,7 +23,7 @@ dremio-services-parent com.dremio.services - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-options diff --git a/services/pom.xml b/services/pom.xml index 2195102f99..5c293630a6 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -22,7 +22,7 @@ com.dremio dremio-root - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.services diff --git a/services/resourcescheduler/pom.xml b/services/resourcescheduler/pom.xml index 025c5ea056..7a6d2c38cc 100644 --- a/services/resourcescheduler/pom.xml +++ b/services/resourcescheduler/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-resourcescheduler Services - Resource Scheduler diff --git a/services/scheduler/pom.xml b/services/scheduler/pom.xml index 4c28d8af14..1263902979 100644 --- a/services/scheduler/pom.xml +++ b/services/scheduler/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-scheduler Services - Task Scheduler diff --git a/services/spill/pom.xml b/services/spill/pom.xml index f158c03765..02ccbd2842 100644 --- a/services/spill/pom.xml +++ b/services/spill/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-spill Services - Spill management diff --git a/services/users/pom.xml b/services/users/pom.xml index d67ad6b8c1..db275f80de 100644 --- a/services/users/pom.xml +++ b/services/users/pom.xml @@ -22,7 +22,7 @@ com.dremio.services dremio-services-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 dremio-services-users diff --git a/tools/build-tools/pom.xml b/tools/build-tools/pom.xml index 85860d7bf3..98549305ad 100644 --- a/tools/build-tools/pom.xml +++ b/tools/build-tools/pom.xml @@ -23,7 +23,7 @@ com.dremio.tools dremio-build-tools - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 jar Tools - Build tools diff --git a/tools/fmpp-maven-plugin/pom.xml b/tools/fmpp-maven-plugin/pom.xml index 85b7bafbf1..e0087d94e9 100644 --- a/tools/fmpp-maven-plugin/pom.xml +++ b/tools/fmpp-maven-plugin/pom.xml @@ -22,7 +22,7 @@ com.dremio.tools dremio-tools-parent - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.tools diff --git a/tools/pom.xml b/tools/pom.xml index 00154b9622..926c2982c6 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -24,7 +24,7 @@ dremio-root com.dremio - 3.1.4-201902210617130445-0d64760 + 3.1.6-201903070042400578-fdcd3a8 com.dremio.tools