Merge branch 'main' into srr-schema-validation-dev-only

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -288,12 +288,76 @@ const getPipeline = (filename: string, removeSteps = true) => {
       pipeline.push(
         getPipeline('.buildkite/pipelines/pull_request/security_solution/entity_analytics.yml')
       );
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution/explore.yml'));
       pipeline.push(
         getPipeline('.buildkite/pipelines/pull_request/security_solution/rule_management.yml')
       );
     }
 
+    if (
+      (await doAnyChangesMatch([
+        /^package.json/,
+        /^src\/platform\/packages\/shared\/kbn-discover-utils/,
+        /^packages\/kbn-doc-links/,
+        /^src\/platform\/packages\/shared\/kbn-dom-drag-drop/,
+        /^src\/platform\/packages\/shared\/kbn-es-query/,
+        /^src\/platform\/packages\/shared\/kbn-i18n/,
+        /^src\/platform\/packages\/shared\/kbn-i18n-react/,
+        /^src\/platform\/packages\/shared\/kbn-grouping/,
+        /^src\/platform\/packages\/shared\/kbn-resizable-layout/,
+        /^src\/platform\/packages\/shared\/kbn-rison/,
+        /^src\/platform\/packages\/shared\/kbn-rule-data-utils/,
+        /^src\/platform\/packages\/shared\/kbn-safer-lodash-set/,
+        /^src\/platform\/packages\/shared\/kbn-search-types/,
+        /^packages\/kbn-securitysolution-.*/,
+        /^src\/platform\/packages\/shared\/kbn-securitysolution-ecs/,
+        /^x-pack\/solutions\/security\/packages\/kbn-securitysolution-io-ts-alerting-types/,
+        /^x-pack\/solutions\/security\/packages\/kbn-securitysolution-io-ts-list-types/,
+        /^x-pack\/solutions\/security\/packages\/kbn-securitysolution-list-hooks/,
+        /^x-pack\/solutions\/security\/packages\/kbn-securitysolution-t-grid/,
+        /^src\/platform\/packages\/shared\/kbn-ui-theme/,
+        /^src\/platform\/packages\/shared\/kbn-utility-types/,
+        /^packages\/react/,
+        /^packages\/shared-ux/,
+        /^src\/core/,
+        /^src\/platform\/plugins\/shared\/charts/,
+        /^src\/platform\/plugins\/shared\/controls/,
+        /^src\/platform\/plugins\/shared\/dashboard/,
+        /^src\/platform\/plugins\/shared\/data/,
+        /^src\/platform\/plugins\/shared\/data_views/,
+        /^src\/platform\/plugins\/shared\/discover/,
+        /^src\/platform\/plugins\/shared\/field_formats/,
+        /^src\/platform\/plugins\/shared\/inspector/,
+        /^src\/platform\/plugins\/shared\/kibana_react/,
+        /^src\/platform\/plugins\/shared\/kibana_utils/,
+        /^src\/platform\/plugins\/shared\/saved_search/,
+        /^src\/platform\/plugins\/shared\/ui_actions/,
+        /^src\/platform\/plugins\/shared\/unified_histogram/,
+        /^src\/platform\/plugins\/shared\/unified_search/,
+        /^x-pack\/platform\/packages\/shared\/kbn-elastic-assistant/,
+        /^x-pack\/platform\/packages\/shared\/kbn-elastic-assistant-common/,
+        /^x-pack\/solutions\/security\/packages/,
+        /^x-pack\/platform\/plugins\/shared\/alerting/,
+        /^x-pack\/platform\/plugins\/shared\/cases/,
+        /^x-pack\/platform\/plugins\/shared\/data_views\/common/,
+        /^x-pack\/solutions\/security\/plugins\/elastic_assistant/,
+        /^x-pack\/solutions\/security\/plugins\/lists/,
+        /^x-pack\/platform\/plugins\/shared\/rule_registry\/common/,
+        /^x-pack\/solutions\/security\/plugins\/security_solution/,
+        /^x-pack\/solutions\/security\/plugins\/security_solution_ess/,
+        /^x-pack\/solutions\/security\/plugins\/security_solution_serverless/,
+        /^x-pack\/platform\/plugins\/shared\/task_manager/,
+        /^x-pack\/solutions\/security\/plugins\/threat_intelligence/,
+        /^x-pack\/platform\/plugins\/shared\/timelines/,
+        /^x-pack\/platform\/plugins\/shared\/triggers_actions_ui/,
+        /^x-pack\/platform\/plugins\/shared\/usage_collection\/public/,
+        /^x-pack\/test\/functional\/es_archives\/security_solution/,
+        /^x-pack\/test\/security_solution_cypress/,
+      ])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/security_solution/explore.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([
         /^package.json/,

config/serverless.es.yml

@@ -113,7 +113,7 @@ xpack.ml.compatibleModuleType: 'search'
 data_visualizer.resultLinks.fileBeat.enabled: false
 
 # Search Notebooks
-xpack.search.notebooks.catalog.url: https://elastic-enterprise-search.s3.us-east-2.amazonaws.com/serverless/catalog.json
+xpack.searchNotebooks.catalog.url: https://elastic-enterprise-search.s3.us-east-2.amazonaws.com/serverless/catalog.json
 
 # Semantic text UI
 

config/serverless.oblt.yml

@@ -6,7 +6,7 @@ xpack.cloudSecurityPosture.enabled: false
 xpack.infra.enabled: true
 xpack.uptime.enabled: true
 xpack.securitySolution.enabled: false
-xpack.search.notebooks.enabled: false
+xpack.searchNotebooks.enabled: false
 xpack.searchPlayground.enabled: false
 xpack.searchInferenceEndpoints.enabled: false
 xpack.searchIndices.enabled: false

config/serverless.security.yml

@@ -7,7 +7,7 @@ xpack.infra.enabled: false
 xpack.observabilityLogsExplorer.enabled: false
 xpack.observability.enabled: false
 xpack.observabilityAIAssistant.enabled: false
-xpack.search.notebooks.enabled: false
+xpack.searchNotebooks.enabled: false
 xpack.searchPlayground.enabled: false
 xpack.searchInferenceEndpoints.enabled: false
 xpack.inventory.enabled: false

package.json

@@ -1258,7 +1258,7 @@
     "rxjs": "^7.8.1",
     "safe-squel": "^5.12.5",
     "seedrandom": "^3.0.5",
-    "semver": "^7.6.3",
+    "semver": "^7.7.0",
     "set-value": "^4.1.0",
     "snakecase-keys": "^8.0.0",
     "source-map-support": "^0.5.19",

src/platform/packages/shared/kbn-data-view-utils/index.ts

@@ -9,5 +9,6 @@
 
 export * from './src/constants';
 export { convertDatatableColumnToDataViewFieldSpec } from './src/utils/convert_to_data_view_field_spec';
+export { getDataViewFieldOrCreateFromColumnMeta } from './src/utils/get_data_view_field_or_create';
 export { createRegExpPatternFrom } from './src/utils/create_regexp_pattern_from';
 export { testPatternAgainstAllowedList } from './src/utils/test_pattern_against_allowed_list';

src/platform/packages/shared/kbn-data-view-utils/src/utils/get_data_view_field_or_create.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { isEqual } from 'lodash';
+import type { DataView } from '@kbn/data-views-plugin/public';
+import type { DatatableColumnMeta } from '@kbn/expressions-plugin/common';
+import { convertDatatableColumnToDataViewFieldSpec } from './convert_to_data_view_field_spec';
+
+export const getDataViewFieldOrCreateFromColumnMeta = ({
+  dataView,
+  fieldName,
+  columnMeta,
+}: {
+  dataView: DataView;
+  fieldName: string;
+  columnMeta?: DatatableColumnMeta; // based on ES|QL query
+}) => {
+  const dataViewField = dataView.fields.getByName(fieldName);
+
+  if (!columnMeta) {
+    return dataViewField;
+  }
+
+  const fieldSpecFromColumnMeta = convertDatatableColumnToDataViewFieldSpec({
+    name: fieldName,
+    id: fieldName,
+    meta: columnMeta,
+  });
+
+  if (
+    !dataViewField ||
+    dataViewField.type !== fieldSpecFromColumnMeta.type ||
+    !isEqual(dataViewField.esTypes, fieldSpecFromColumnMeta.esTypes)
+  ) {
+    return dataView.fields.create(fieldSpecFromColumnMeta);
+  }
+
+  return dataViewField;
+};

src/platform/packages/shared/kbn-discover-utils/src/__mocks__/data_view.ts

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { DataView, DataViewField } from '@kbn/data-views-plugin/public';
+import { DataView, DataViewField, FieldSpec } from '@kbn/data-views-plugin/public';
 
 export const shallowMockedFields = [
   {
@@ -101,6 +101,10 @@ export const buildDataViewMock = ({
     return dataViewFields;
   };
 
+  dataViewFields.create = (spec: FieldSpec) => {
+    return new DataViewField(spec);
+  };
+
   const dataView = {
     id: `${name}-id`,
     title: `${name}-title`,

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts

@@ -16,6 +16,13 @@ import { getCommandDefinition } from '../shared/helpers';
 import { getCommandSignature } from '../definitions/helpers';
 import { buildDocumentation } from './documentation_util';
 
+const techPreviewLabel = i18n.translate(
+  'kbn-esql-validation-autocomplete.esql.autocomplete.techPreviewLabel',
+  {
+    defaultMessage: `Technical Preview`,
+  }
+);
+
 export function getAssignmentDefinitionCompletitionItem() {
   const assignFn = builtinFunctions.find(({ name }) => name === '=')!;
   return getOperatorSuggestion(assignFn);
@@ -32,7 +39,6 @@ export const getCommandAutocompleteDefinitions = (
     }
 
     const commandDefinition = getCommandDefinition(command.name);
-    const commandSignature = getCommandSignature(commandDefinition);
     const label = commandDefinition.name.toUpperCase();
     const text = commandDefinition.signature.params.length
       ? `${commandDefinition.name.toUpperCase()} $0`
@@ -45,12 +51,17 @@ export const getCommandAutocompleteDefinitions = (
     ];
 
     for (const type of types) {
+      let detail = type.description || commandDefinition.description;
+      if (commandDefinition.preview) {
+        detail = `[${techPreviewLabel}] ${detail}`;
+      }
+      const commandSignature = getCommandSignature(commandDefinition, type.name);
       const suggestion: SuggestionRawDefinition = {
         label: type.name ? `${type.name.toLocaleUpperCase()} ${label}` : label,
         text: type.name ? `${type.name.toLocaleUpperCase()} ${text}` : text,
         asSnippet: true,
         kind: 'Method',
-        detail: type.description || commandDefinition.description,
+        detail,
         documentation: {
           value: buildDocumentation(commandSignature.declaration, commandSignature.examples),
         },

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/commands.ts

@@ -485,6 +485,7 @@ export const commandDefinitions: Array<CommandDefinition<any>> = [
     examples: ['row a=[1,2,3] | mv_expand a'],
     options: [],
     modes: [],
+    preview: true,
     signature: {
       multipleParams: false,
       params: [{ name: 'column', type: 'column', innerTypes: ['any'] }],
@@ -557,19 +558,20 @@ export const commandDefinitions: Array<CommandDefinition<any>> = [
     description: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.joinDoc', {
       defaultMessage: 'Join table with another table.',
     }),
+    preview: true,
     examples: [
       '… | LOOKUP JOIN lookup_index ON join_field',
       // TODO: Uncomment when other join types are implemented
       // '… | <LEFT | RIGHT | LOOKUP> JOIN index ON index.field = index2.field',
       // '… | <LEFT | RIGHT | LOOKUP> JOIN index AS alias ON index.field = index2.field',
       // '… | <LEFT | RIGHT | LOOKUP> JOIN index AS alias ON index.field = index2.field, index.field2 = index2.field2',
     ],
-    options: [],
     modes: [],
     signature: {
-      multipleParams: false,
+      multipleParams: true,
       params: [{ name: 'index', type: 'source', wildcards: true }],
     },
+    options: [onOption],
     suggest: suggestForJoin,
   },
 ];

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/helpers.ts

@@ -57,10 +57,14 @@ function handleAdditionalArgs(
 
 export function getCommandSignature(
   { name, signature, options, examples }: CommandDefinition<string>,
+  typeName?: string,
   { withTypes }: { withTypes: boolean } = { withTypes: true }
 ) {
+  const commandName = typeName
+    ? `${typeName.toUpperCase()} ${name.toUpperCase()}`
+    : name.toUpperCase();
   return {
-    declaration: `${name.toUpperCase()} ${printCommandArguments(signature, withTypes)} ${(
+    declaration: `${commandName} ${printCommandArguments(signature, withTypes)} ${(
       options || []
     ).map(
       (option) =>

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/types.ts

@@ -194,6 +194,10 @@ export interface CommandBaseDefinition<CommandName extends string> {
 
   alias?: string;
   description: string;
+  /**
+   * Displays a Technical preview label in the autocomplete
+   */
+  preview?: boolean;
   /**
    * Whether to show or hide in autocomplete suggestion list
    */

src/platform/packages/shared/kbn-unified-data-table/__mocks__/table_context.ts

@@ -36,6 +36,7 @@ const buildTableContext = (dataView: DataView, rows: DataTableRecord[]): DataTab
         fieldFormats: servicesMock.fieldFormats,
         rows,
         dataView,
+        columnsMeta: undefined,
         options,
       }),
   };