Skip to content

Commit

Permalink
Remove seed generation task and seed from YAML files (airbytehq#5335)
Browse files Browse the repository at this point in the history
## What
- This is the first PR for airbytehq#4890.
  - This PR does not remove the config volume.
  - This PR does not mount the directories for the local connectors.
- Resolves airbytehq#5373.

## How
-  Previously the seed container copies the configs to the storage root, it may take some time for the operation to complete and for the `CONFIG_DIR` to show up. So we cannot infer anything based on the existence of this directory. Now this seed generation step has been removed. So we can tell immediately whether `CONFIG_DIR` exists or not.
  - If `CONFIG_DIR` exists, it means the user has just migrated Airbyte from an old version that uses this file system config persistence.
  - Otherwise, we can seed the config persistence from the YAML files.
  • Loading branch information
tuliren authored Aug 17, 2021
1 parent 3e89961 commit 79b8fd5
Show file tree
Hide file tree
Showing 27 changed files with 329 additions and 495 deletions.
39 changes: 0 additions & 39 deletions airbyte-config/init/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -7,42 +7,3 @@ dependencies {

implementation project(':airbyte-config:models')
}

// generate seed for each yaml file.
task generateSeed {
def seeds = [
[
"sourceDefinitionId",
new File(project.projectDir, '/src/main/resources/seed/source_definitions.yaml'),
new File(project.projectDir, '/src/main/resources/config/STANDARD_SOURCE_DEFINITION')
],
[
"destinationDefinitionId",
new File(project.projectDir, '/src/main/resources/seed/destination_definitions.yaml'),
new File(project.projectDir, '/src/main/resources/config/STANDARD_DESTINATION_DEFINITION')
],
]
seeds.each{val ->
def name = val[0]
def taskName = "generateSeed$name"
dependsOn taskName
task "$taskName"(type: JavaExec) {
classpath = sourceSets.main.runtimeClasspath

main = 'io.airbyte.config.init.SeedRepository'

// arguments to pass to the application
args '--id-name'
args val[0]
args '--input-path'
args val[1]
args '--output-path'
args val[2]
}
}
}

// we only want to attempt generateSeed if tests have passed.
generateSeed.dependsOn(check)
generateSeed.dependsOn(assemble)
build.dependsOn(generateSeed)
10 changes: 0 additions & 10 deletions airbyte-config/init/scripts/copy_seed_data.sh

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* MIT License
*
* Copyright (c) 2020 Airbyte
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package io.airbyte.config.init;

public enum SeedType {

STANDARD_SOURCE_DEFINITION("/seed/source_definitions.yaml", "sourceDefinitionId"),
STANDARD_DESTINATION_DEFINITION("/seed/destination_definitions.yaml", "destinationDefinitionId");

final String resourcePath;
// ID field name
final String idName;

SeedType(String resourcePath, String idName) {
this.resourcePath = resourcePath;
this.idName = idName;
}

public String getResourcePath() {
return resourcePath;
}

public String getIdName() {
return idName;
}

}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,13 @@ public static ConfigPersistence getDbPersistence(Configs configs) throws IOExcep
* Otherwise, seed the database from the yaml files.
*/
ConfigPersistence create() throws IOException, InterruptedException {
// Uncomment this branch in a future version when config volume is removed.
// if (configs.getConfigRoot() == null) {
// return getDbPersistenceWithYamlSeed();
// }
return getDbPersistenceWithFileSeed();
if (FileSystemConfigPersistence.hasExistingConfigs(configs.getConfigRoot())) {
LOGGER.info("There is existing local config directory");
return getDbPersistenceWithFileSeed();
} else {
LOGGER.info("There is no existing local config directory");
return getDbPersistenceWithYamlSeed();
}
}

/**
Expand All @@ -81,7 +83,7 @@ ConfigPersistence create() throws IOException, InterruptedException {
*/
ConfigPersistence getDbPersistenceWithYamlSeed() throws IOException {
LOGGER.info("Creating db-based config persistence, and loading initial seed from YAML files");
ConfigPersistence seedConfigPersistence = new YamlSeedConfigPersistence();
ConfigPersistence seedConfigPersistence = YamlSeedConfigPersistence.get();
return getDbPersistence(seedConfigPersistence);
}

Expand Down
Loading

0 comments on commit 79b8fd5

Please sign in to comment.