🐛 octavia-cli: fix sync_catalog in generated connections (airbytehq#1…

…2704)

octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected.yaml

@@ -21,44 +21,14 @@ configuration:
   sync_catalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed.
     streams:
       - config:
-          aliasName: aliasMock
-          cursorField: []
-          destinationSyncMode: append
-          primaryKey: []
+          alias_name: pokemon
+          destination_sync_mode: append
           selected: true
-          syncMode: full_refresh
+          sync_mode: full_refresh
         stream:
-          defaultCursorField:
+          default_cursor_field:
             - foo
-          jsonSchema:
-            $schema: http://json-schema.org/draft-07/schema#
-            properties:
-              foo:
-                type: number
-          name: stream_1
-          namespace: null
-          sourceDefinedCursor: null
-          sourceDefinedPrimaryKey: []
-          supportedSyncModes:
+          json_schema: {}
+          name: my_stream
+          supported_sync_modes:
             - full_refresh
-      - config:
-          aliasName: aliasMock
-          cursorField: []
-          destinationSyncMode: append
-          primaryKey: []
-          selected: true
-          syncMode: full_refresh
-        stream:
-          defaultCursorField: []
-          jsonSchema:
-            $schema: http://json-schema.org/draft-07/schema#
-            properties:
-              bar:
-                type: number
-          name: stream_2
-          namespace: null
-          sourceDefinedCursor: null
-          sourceDefinedPrimaryKey: []
-          supportedSyncModes:
-            - full_refresh
-            - incremental

octavia-cli/integration_tests/test_generate/test_renderers.py

@@ -7,6 +7,12 @@
 
 import pytest
 import yaml
+from airbyte_api_client.model.airbyte_catalog import AirbyteCatalog
+from airbyte_api_client.model.airbyte_stream import AirbyteStream
+from airbyte_api_client.model.airbyte_stream_and_configuration import AirbyteStreamAndConfiguration
+from airbyte_api_client.model.airbyte_stream_configuration import AirbyteStreamConfiguration
+from airbyte_api_client.model.destination_sync_mode import DestinationSyncMode
+from airbyte_api_client.model.sync_mode import SyncMode
 from octavia_cli.generate.renderers import ConnectionRenderer, ConnectorSpecificationRenderer
 
 pytestmark = pytest.mark.integration
@@ -89,65 +95,12 @@ def test_expected_output_connector_specification_renderer(
 
 
 def test_expected_output_connection_renderer(octavia_tmp_project_directory, mocker):
-    mock_source = mocker.Mock(
-        resource_id="my_source_id",
-        catalog={
-            "streams": [
-                {
-                    "stream": {
-                        "name": "stream_1",
-                        "jsonSchema": {
-                            "$schema": "http://json-schema.org/draft-07/schema#",
-                            "properties": {
-                                "foo": {
-                                    "type": "number",
-                                }
-                            },
-                        },
-                        "supportedSyncModes": ["full_refresh"],
-                        "sourceDefinedCursor": None,
-                        "defaultCursorField": ["foo"],
-                        "sourceDefinedPrimaryKey": [],
-                        "namespace": None,
-                    },
-                    "config": {
-                        "syncMode": "full_refresh",
-                        "cursorField": [],
-                        "destinationSyncMode": "append",
-                        "primaryKey": [],
-                        "aliasName": "aliasMock",
-                        "selected": True,
-                    },
-                },
-                {
-                    "stream": {
-                        "name": "stream_2",
-                        "jsonSchema": {
-                            "$schema": "http://json-schema.org/draft-07/schema#",
-                            "properties": {
-                                "bar": {
-                                    "type": "number",
-                                }
-                            },
-                        },
-                        "supportedSyncModes": ["full_refresh", "incremental"],
-                        "sourceDefinedCursor": None,
-                        "defaultCursorField": [],
-                        "sourceDefinedPrimaryKey": [],
-                        "namespace": None,
-                    },
-                    "config": {
-                        "syncMode": "full_refresh",
-                        "cursorField": [],
-                        "destinationSyncMode": "append",
-                        "primaryKey": [],
-                        "aliasName": "aliasMock",
-                        "selected": True,
-                    },
-                },
-            ]
-        },
+    stream = AirbyteStream(default_cursor_field=["foo"], json_schema={}, name="my_stream", supported_sync_modes=[SyncMode("full_refresh")])
+    config = AirbyteStreamConfiguration(
+        alias_name="pokemon", selected=True, destination_sync_mode=DestinationSyncMode("append"), sync_mode=SyncMode("full_refresh")
     )
+    catalog = AirbyteCatalog([AirbyteStreamAndConfiguration(stream=stream, config=config)])
+    mock_source = mocker.Mock(resource_id="my_source_id", catalog=catalog)
     mock_destination = mocker.Mock(resource_id="my_destination_id")
 
     renderer = ConnectionRenderer("my_new_connection", mock_source, mock_destination)

octavia-cli/octavia_cli/generate/renderers.py

@@ -7,6 +7,7 @@
 from typing import Any, Callable, List
 
 import yaml
+from airbyte_api_client.model.airbyte_catalog import AirbyteCatalog
 from jinja2 import Environment, PackageLoader, Template, select_autoescape
 from octavia_cli.apply import resources
 
@@ -241,16 +242,16 @@ def __init__(self, connection_name: str, source: resources.Source, destination:
         self.destination = destination
 
     @staticmethod
-    def catalog_to_yaml(catalog: dict) -> str:
+    def catalog_to_yaml(catalog: AirbyteCatalog) -> str:
         """Convert the source catalog to a YAML string.
 
         Args:
-            catalog (dict): Source's catalog.
+            catalog (AirbyteCatalog): Source's catalog.
 
         Returns:
             str: Catalog rendered as yaml.
         """
-        return yaml.dump(catalog, Dumper=CatalogDumper, default_flow_style=False)
+        return yaml.dump(catalog.to_dict(), Dumper=CatalogDumper, default_flow_style=False)
 
     def _render(self) -> str:
         yaml_catalog = self.catalog_to_yaml(self.source.catalog)

octavia-cli/unit_tests/test_generate/test_renderers.py

@@ -5,7 +5,14 @@
 from unittest.mock import mock_open, patch
 
 import pytest
-from octavia_cli.generate import renderers
+import yaml
+from airbyte_api_client.model.airbyte_catalog import AirbyteCatalog
+from airbyte_api_client.model.airbyte_stream import AirbyteStream
+from airbyte_api_client.model.airbyte_stream_and_configuration import AirbyteStreamAndConfiguration
+from airbyte_api_client.model.airbyte_stream_configuration import AirbyteStreamConfiguration
+from airbyte_api_client.model.destination_sync_mode import DestinationSyncMode
+from airbyte_api_client.model.sync_mode import SyncMode
+from octavia_cli.generate import renderers, yaml_dumpers
 
 
 class TestFieldToRender:
@@ -268,9 +275,15 @@ def test_init(self, mock_source, mock_destination):
         assert connection_renderer.destination == mock_destination
 
     def test_catalog_to_yaml(self, mocker):
-        catalog = {"camelCase": "camelCase", "snake_case": "camelCase", "myArray": ["a", "b"]}
+        stream = AirbyteStream(
+            default_cursor_field=["foo"], json_schema={}, name="my_stream", supported_sync_modes=[SyncMode("full_refresh")]
+        )
+        config = AirbyteStreamConfiguration(
+            alias_name="pokemon", selected=True, destination_sync_mode=DestinationSyncMode("append"), sync_mode=SyncMode("full_refresh")
+        )
+        catalog = AirbyteCatalog([AirbyteStreamAndConfiguration(stream=stream, config=config)])
         yaml_catalog = renderers.ConnectionRenderer.catalog_to_yaml(catalog)
-        assert yaml_catalog == "camelCase: camelCase\nmyArray:\n  - a\n  - b\nsnake_case: camelCase\n"
+        assert yaml_catalog == yaml.dump(catalog.to_dict(), Dumper=yaml_dumpers.CatalogDumper, default_flow_style=False)
 
     def test_write_yaml(self, mocker, mock_source, mock_destination):
         mocker.patch.object(renderers.ConnectionRenderer, "_get_output_path")