Add option to disable migration via disableKbSemanticTextMigration

x-pack/platform/plugins/shared/observability_ai_assistant/server/config.ts

@@ -11,6 +11,7 @@ export const config = schema.object({
   enabled: schema.boolean({ defaultValue: true }),
   scope: schema.maybe(schema.oneOf([schema.literal('observability'), schema.literal('search')])),
   enableKnowledgeBase: schema.boolean({ defaultValue: true }),
+  disableKbSemanticTextMigration: schema.boolean({ defaultValue: false }),
 });
 
 export type ObservabilityAIAssistantConfig = TypeOf<typeof config>;

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/knowledge_base_service/reindex_knowledge_base.ts

@@ -52,7 +52,7 @@ export async function reIndexKnowledgeBase({
     await esClient.asInternalUser.indices.create({ index: tempIndex });
 
     // Perform reindex to temporary index
-    logger.debug(`Reindexing knowledge base to temporary index "${tempIndex}"...`);
+    logger.debug(`Re-indexing knowledge base to temporary index "${tempIndex}"...`);
     await esClient.asInternalUser.reindex({
       body: {
         source: { index: originalIndex },
@@ -68,7 +68,7 @@ export async function reIndexKnowledgeBase({
     await createKbConcreteIndex({ logger, esClient });
 
     // Perform reindex back to original index
-    logger.debug(`Reindexing knowledge base back to original index "${originalIndex}"...`);
+    logger.debug(`Re-indexing knowledge base back to original index "${originalIndex}"...`);
     await esClient.asInternalUser.reindex({
       body: {
         source: { index: tempIndex },
@@ -86,7 +86,7 @@ export async function reIndexKnowledgeBase({
       'Re-indexing knowledge base completed successfully. Semantic text field is now supported.'
     );
   } catch (error) {
-    throw new Error(`Failed to re-index knowledge base: ${error.message}`);
+    throw new Error(`Failed to reindex knowledge base: ${error.message}`);
   }
 }
 

x-pack/platform/plugins/shared/observability_ai_assistant/server/service/task_manager_definitions/register_kb_semantic_text_migration_task.ts

@@ -59,7 +59,10 @@ export async function registerKbSemanticTextMigrationTask({
                 return;
               }
 
-              if (true) {
+              if (config.disableKbSemanticTextMigration) {
+                logger.debug(
+                  'Semantic text migration is disabled via "disableKbSemanticTextMigration=true". Skipping migration.'
+                );
                 return;
               }
 

x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/knowledge_base/knowledge_base_reindex.spec.ts

@@ -24,75 +24,12 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
   const retry = getService('retry');
   const log = getService('log');
 
-  async function getKbIndexCreatedVersion() {
-    const indexSettings = await es.indices.getSettings({
-      index: resourceNames.concreteIndexName.kb,
-    });
-
-    const { settings } = Object.values(indexSettings)[0];
-    return parseInt(settings?.index?.version?.created ?? '', 10);
-  }
-
-  async function deleteKbIndex() {
-    log.debug('Deleting KB index');
-
-    await es.indices.delete(
-      { index: resourceNames.concreteIndexName.kb, ignore_unavailable: true },
-      { ignore: [404] }
-    );
-  }
-
-  async function restoreKbSnapshot() {
-    log.debug(`Restoring snapshot of ${resourceNames.concreteIndexName.kb}`);
-    const snapshotRepoName = 'snapshot-repo-8-10';
-    const snapshotName = 'my_snapshot';
-    await es.snapshot.createRepository({
-      name: snapshotRepoName,
-      repository: {
-        type: 'fs',
-        settings: { location: AI_ASSISTANT_SNAPSHOT_REPO_PATH },
-      },
-    });
-
-    await es.snapshot.restore({
-      repository: snapshotRepoName,
-      snapshot: snapshotName,
-      wait_for_completion: true,
-      body: {
-        indices: resourceNames.concreteIndexName.kb,
-      },
-    });
-
-    await es.snapshot.deleteRepository({ name: snapshotRepoName });
-  }
-
-  async function createOrUpdateIndexAssets() {
-    const { status } = await observabilityAIAssistantAPIClient.editor({
-      endpoint: 'POST /internal/observability_ai_assistant/index_assets',
-    });
-    expect(status).to.be(200);
-  }
-
-  async function runKbSemanticTextMigration() {
-    const { status } = await observabilityAIAssistantAPIClient.editor({
-      endpoint: 'POST /internal/observability_ai_assistant/kb/migrations/kb_semantic_text',
-    });
-    expect(status).to.be(200);
-  }
-
-  describe('indices created before 8.11 should be re-indexed in order to support sparse_vector fields', function () {
+  describe('when the knowledge base index was created before 8.11', function () {
     // Intentionally skipped in all serverless environnments (local and MKI)
     // because the migration scenario being tested is not relevant to MKI and Serverless.
     this.tags(['skipServerless']);
 
     before(async () => {
-      try {
-        await deleteKnowledgeBaseModel(ml);
-        await deleteInferenceEndpoint({ es });
-      } catch (e) {
-        // ignore
-      }
-
       await importTinyElserModel(ml);
       await setupKnowledgeBase(observabilityAIAssistantAPIClient);
       await waitForKnowledgeBaseReady({ observabilityAIAssistantAPIClient, log, retry });
@@ -105,14 +42,13 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
     });
 
     after(async () => {
-      // await deleteKbIndex();
-      // await createKbIndex();
-      // await createOrUpdateIndexAssets();
-      // await deleteKnowledgeBaseModel(ml);
-      // await deleteInferenceEndpoint({ es });
+      await deleteKbIndex();
+      await createOrUpdateIndexAssets();
+      await deleteKnowledgeBaseModel(ml);
+      await deleteInferenceEndpoint({ es });
     });
 
-    it('has knowledge base index created before 8.11', async () => {
+    it('has an index created version earlier than 8.11', async () => {
       await retry.try(async () => {
         expect(await getKbIndexCreatedVersion()).to.be.lessThan(8110000);
       });
@@ -131,7 +67,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
       });
     }
 
-    it('cannot add new entries to KB on the first attempt', async () => {
+    it('cannot add new entries to KB', async () => {
       const { status, body } = await createKnowledgeBaseEntry();
 
       // @ts-expect-error
@@ -151,4 +87,60 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
       });
     });
   });
+
+  async function getKbIndexCreatedVersion() {
+    const indexSettings = await es.indices.getSettings({
+      index: resourceNames.concreteIndexName.kb,
+    });
+
+    const { settings } = Object.values(indexSettings)[0];
+    return parseInt(settings?.index?.version?.created ?? '', 10);
+  }
+
+  async function deleteKbIndex() {
+    log.debug('Deleting KB index');
+
+    await es.indices.delete(
+      { index: resourceNames.concreteIndexName.kb, ignore_unavailable: true },
+      { ignore: [404] }
+    );
+  }
+
+  async function restoreKbSnapshot() {
+    log.debug(`Restoring snapshot of ${resourceNames.concreteIndexName.kb}`);
+    const snapshotRepoName = 'snapshot-repo-8-10';
+    const snapshotName = 'my_snapshot';
+    await es.snapshot.createRepository({
+      name: snapshotRepoName,
+      repository: {
+        type: 'fs',
+        settings: { location: AI_ASSISTANT_SNAPSHOT_REPO_PATH },
+      },
+    });
+
+    await es.snapshot.restore({
+      repository: snapshotRepoName,
+      snapshot: snapshotName,
+      wait_for_completion: true,
+      body: {
+        indices: resourceNames.concreteIndexName.kb,
+      },
+    });
+
+    await es.snapshot.deleteRepository({ name: snapshotRepoName });
+  }
+
+  async function createOrUpdateIndexAssets() {
+    const { status } = await observabilityAIAssistantAPIClient.editor({
+      endpoint: 'POST /internal/observability_ai_assistant/index_assets',
+    });
+    expect(status).to.be(200);
+  }
+
+  async function runKbSemanticTextMigration() {
+    const { status } = await observabilityAIAssistantAPIClient.editor({
+      endpoint: 'POST /internal/observability_ai_assistant/kb/migrations/kb_semantic_text',
+    });
+    expect(status).to.be(200);
+  }
 }