Add supportsDBT and supportsNormalization to API objects (airbytehq#4031

)

* Add supportsDBT and supportsNormalization to API objects

airbyte-api/src/main/openapi/config.yaml

@@ -1842,6 +1842,10 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/DestinationSyncMode"
+        supportsDbt:
+          type: boolean
+        supportsNormalization:
+          type: boolean
     # DESTINATION
     DestinationId:
       type: string

airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py

@@ -114,8 +114,8 @@ class Config:
         description="ConnectorDefinition specific blob. Must be a valid JSON string.",
     )
     supportsIncremental: Optional[bool] = Field(None, description="If the connector supports incremental mode or not.")
-    supportsNormalization: Optional[bool] = Field(None, description="If the connector supports normalization or not.")
-    supportsDBT: Optional[bool] = Field(None, description="If the connector supports DBT or not.")
+    supportsNormalization: Optional[bool] = Field(False, description="If the connector supports normalization or not.")
+    supportsDBT: Optional[bool] = Field(False, description="If the connector supports DBT or not.")
     supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field(
         None, description="List of destination sync modes supported by the connector"
     )

airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py

@@ -114,8 +114,8 @@ class Config:
         description="ConnectorDefinition specific blob. Must be a valid JSON string.",
     )
     supportsIncremental: Optional[bool] = Field(None, description="If the connector supports incremental mode or not.")
-    supportsNormalization: Optional[bool] = Field(None, description="If the connector supports normalization or not.")
-    supportsDBT: Optional[bool] = Field(None, description="If the connector supports DBT or not.")
+    supportsNormalization: Optional[bool] = Field(False, description="If the connector supports normalization or not.")
+    supportsDBT: Optional[bool] = Field(False, description="If the connector supports DBT or not.")
     supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = Field(
         None, description="List of destination sync modes supported by the connector"
     )

airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json

@@ -1,6 +1,8 @@
 {
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle",
   "supportsIncremental": true,
+  "supportsNormalization": false,
+  "supportsDBT": true,
   "supported_destination_sync_modes": ["overwrite", "append"],
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",

airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json

@@ -1,6 +1,8 @@
 {
   "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3",
   "supportsIncremental": true,
+  "supportsNormalization": false,
+  "supportsDBT": false,
   "supported_destination_sync_modes": ["overwrite", "append"],
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",

airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml

@@ -236,13 +236,15 @@ definitions:
         description: If the connector supports incremental mode or not.
         type: boolean
       # Destination Connectors Properties
-      #Normalization is currently implemented using dbt so it requires `supportsDBT` to be true for this to be true.
+      # Normalization is currently implemented using dbt so it requires `supportsDBT` to be true for this to be true.
       supportsNormalization:
         description: If the connector supports normalization or not.
         type: boolean
+        default: false
       supportsDBT:
         description: If the connector supports DBT or not.
         type: boolean
+        default: false
       supported_destination_sync_modes:
         description: List of destination sync modes supported by the connector
         type: array

airbyte-server/src/main/java/io/airbyte/server/handlers/OperationsHandler.java

@@ -50,6 +50,7 @@
 import io.airbyte.config.persistence.ConfigRepository;
 import io.airbyte.validation.json.JsonValidationException;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
 import java.util.function.Supplier;
@@ -198,10 +199,10 @@ public void deleteOperationsForConnection(UUID connectionId, List<UUID> deleteOp
     deleteOperationsForConnection(standardSync, deleteOperationIds);
   }
 
-  public void deleteOperationsForConnection(StandardSync standardSync, List<UUID> deleteOperationIds)
+  public void deleteOperationsForConnection(final StandardSync standardSync, final List<UUID> deleteOperationIds)
       throws JsonValidationException, ConfigNotFoundException, IOException {
     final List<StandardSync> allStandardSyncs = configRepository.listStandardSyncs();
-    final List<UUID> operationIds = standardSync.getOperationIds();
+    final List<UUID> operationIds = new ArrayList<>(standardSync.getOperationIds());
     for (UUID operationId : deleteOperationIds) {
       operationIds.remove(operationId);
       boolean sharedOperation = false;

airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java

@@ -274,6 +274,8 @@ public DestinationDefinitionSpecificationRead getDestinationSpecification(Destin
         .supportedDestinationSyncModes(Enums.convertListTo(spec.getSupportedDestinationSyncModes(), DestinationSyncMode.class))
         .connectionSpecification(spec.getConnectionSpecification())
         .documentationUrl(spec.getDocumentationUrl().toString())
+        .supportsNormalization(spec.getSupportsNormalization())
+        .supportsDbt(spec.getSupportsDBT())
         .destinationDefinitionId(destinationDefinitionId);
   }
 

airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java

@@ -112,7 +112,9 @@ void setUp() throws IOException {
     destinationDefinitionSpecificationRead = new DestinationDefinitionSpecificationRead()
         .connectionSpecification(connectorSpecification.getConnectionSpecification())
         .destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId())
-        .documentationUrl(connectorSpecification.getDocumentationUrl().toString());
+        .documentationUrl(connectorSpecification.getDocumentationUrl().toString())
+        .supportsDbt(connectorSpecification.getSupportsDBT())
+        .supportsNormalization(connectorSpecification.getSupportsNormalization());
 
     destinationConnection = DestinationHelpers.generateDestination(standardDestinationDefinition.getDestinationDefinitionId());
 

airbyte-server/src/test/java/io/airbyte/server/handlers/OperationsHandlerTest.java

@@ -62,6 +62,7 @@ class OperationsHandlerTest {
   private Supplier<UUID> uuidGenerator;
 
   private OperationsHandler operationsHandler;
+  private StandardSync standardSync;
   private StandardSyncOperation standardSyncOperation;
 
   @SuppressWarnings("unchecked")
@@ -71,6 +72,7 @@ void setUp() throws IOException {
     uuidGenerator = mock(Supplier.class);
 
     operationsHandler = new OperationsHandler(configRepository, uuidGenerator);
+    standardSync = mock(StandardSync.class);
     standardSyncOperation = new StandardSyncOperation()
         .withOperationId(UUID.randomUUID())
         .withName("presto to hudi")
@@ -205,6 +207,27 @@ void testDeleteOperation() throws JsonValidationException, IOException, ConfigNo
     verify(configRepository).writeStandardSyncOperation(standardSyncOperation.withTombstone(true));
   }
 
+  @Test
+  void testDeleteOperationsForConnection() throws JsonValidationException, IOException, ConfigNotFoundException {
+    final UUID operationId = UUID.randomUUID();
+    final List<UUID> toDelete = List.of(standardSyncOperation.getOperationId(), operationId);
+    final StandardSync sync = new StandardSync()
+        .withConnectionId(UUID.randomUUID())
+        .withOperationIds(List.of(standardSyncOperation.getOperationId(), operationId));
+    when(configRepository.listStandardSyncs()).thenReturn(List.of(
+        sync,
+        new StandardSync()
+            .withConnectionId(UUID.randomUUID())
+            .withOperationIds(List.of(standardSyncOperation.getOperationId()))));
+    final StandardSyncOperation operation = new StandardSyncOperation().withOperationId(operationId);
+    when(configRepository.getStandardSyncOperation(operationId)).thenReturn(operation);
+    when(configRepository.getStandardSyncOperation(standardSyncOperation.getOperationId())).thenReturn(standardSyncOperation);
+
+    operationsHandler.deleteOperationsForConnection(sync, toDelete);
+
+    verify(configRepository).writeStandardSyncOperation(operation.withTombstone(true));
+  }
+
   @Test
   void testEnumConversion() {
     assertTrue(Enums.isCompatible(io.airbyte.api.model.OperatorType.class, io.airbyte.config.StandardSyncOperation.OperatorType.class));

airbyte-server/src/test/java/io/airbyte/server/helpers/ConnectorSpecificationHelpers.java

@@ -42,7 +42,9 @@ public static ConnectorSpecification generateConnectorSpecification() throws IOE
     try {
       return new ConnectorSpecification()
           .withDocumentationUrl(new URI("https://airbyte.io"))
-          .withConnectionSpecification(Jsons.deserialize(Files.readString(path)));
+          .withConnectionSpecification(Jsons.deserialize(Files.readString(path)))
+          .withSupportsDBT(false)
+          .withSupportsNormalization(false);
     } catch (URISyntaxException e) {
       throw new RuntimeException(e);
     }

docs/reference/api/generated-api-html/index.html

@@ -1924,12 +1924,14 @@ <h3 class="field-label">Example data</h3>
     <div class="example-data-content-type">Content-Type: application/json</div>
     <pre class="example"><code>{
   "documentationUrl" : "documentationUrl",
+  "supportsNormalization" : true,
   "connectionSpecification" : {
     "user" : {
       "type" : "string"
     }
   },
   "supportedDestinationSyncModes" : [ null, null ],
+  "supportsDbt" : true,
   "destinationDefinitionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
   "jobInfo" : {
     "createdAt" : 0,
@@ -5489,6 +5491,8 @@ <h3><a name="DestinationDefinitionSpecificationRead"><code>DestinationDefinition
 <div class="param">connectionSpecification (optional)</div><div class="param-desc"><span class="param-type"><a href="#DestinationDefinitionSpecification">DestinationDefinitionSpecification</a></span>  </div>
 <div class="param">jobInfo </div><div class="param-desc"><span class="param-type"><a href="#SynchronousJobRead">SynchronousJobRead</a></span>  </div>
 <div class="param">supportedDestinationSyncModes (optional)</div><div class="param-desc"><span class="param-type"><a href="#DestinationSyncMode">array[DestinationSyncMode]</a></span>  </div>
+<div class="param">supportsDbt (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span>  </div>
+<div class="param">supportsNormalization (optional)</div><div class="param-desc"><span class="param-type"><a href="#boolean">Boolean</a></span>  </div>
     </div>  <!-- field-items -->
   </div>
   <div class="model">