Skip to content

Commit

Permalink
Remove jobs and attempts tables from exported archive (airbytehq#8646)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alafanechere
alafanechere authored Dec 20, 2021
1 parent ff4b83b commit ffbf8bc
Show file tree
Hide file tree
Showing 2 changed files with 0 additions and 85 deletions.
31 changes: 0 additions & 31 deletions airbyte-server/src/main/java/io/airbyte/server/ConfigDumpExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
import com.fasterxml.jackson.databind.JsonNode;
import io.airbyte.commons.io.Archives;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.lang.CloseableConsumer;
import io.airbyte.commons.lang.Exceptions;
import io.airbyte.commons.yaml.Yamls;
import io.airbyte.config.ConfigSchema;
import io.airbyte.config.DestinationConnection;
Expand All @@ -21,9 +19,7 @@
import io.airbyte.scheduler.persistence.JobPersistence;
import io.airbyte.scheduler.persistence.WorkspaceHelper;
import io.airbyte.validation.json.JsonValidationException;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
Expand All @@ -34,7 +30,6 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
Expand All @@ -56,7 +51,6 @@ public class ConfigDumpExporter {

private static final String ARCHIVE_FILE_NAME = "airbyte_config_dump";
private static final String CONFIG_FOLDER_NAME = "airbyte_config";
private static final String DB_FOLDER_NAME = "airbyte_db";
private static final String VERSION_FILE_NAME = "VERSION";
private final ConfigRepository configRepository;
private final JobPersistence jobPersistence;
Expand All @@ -74,7 +68,6 @@ public File dump() {
final File dump = Files.createTempFile(ARCHIVE_FILE_NAME, ".tar.gz").toFile();
exportVersionFile(tempFolder);
dumpConfigsDatabase(tempFolder);
dumpJobsDatabase(tempFolder);

Archives.createArchive(tempFolder, dump.toPath());
return dump;
Expand All @@ -89,30 +82,6 @@ private void exportVersionFile(final Path tempFolder) throws IOException {
FileUtils.writeStringToFile(versionFile, version, Charset.defaultCharset());
}

private void dumpJobsDatabase(final Path parentFolder) throws Exception {
final Map<String, Stream<JsonNode>> tables = jobPersistence.exportDatabase().entrySet().stream()
.collect(Collectors.toMap(e -> e.getKey().name(), Entry::getValue));
Files.createDirectories(parentFolder.resolve(DB_FOLDER_NAME));
for (final Map.Entry<String, Stream<JsonNode>> table : tables.entrySet()) {
final Path tablePath = buildTablePath(parentFolder, table.getKey());
writeTableToArchive(tablePath, table.getValue());
}
}

private void writeTableToArchive(final Path tablePath, final Stream<JsonNode> tableStream) throws Exception {
Files.createDirectories(tablePath.getParent());
final BufferedWriter recordOutputWriter = new BufferedWriter(new FileWriter(tablePath.toFile()));
final CloseableConsumer<JsonNode> recordConsumer = Yamls.listWriter(recordOutputWriter);
tableStream.forEach(row -> Exceptions.toRuntime(() -> recordConsumer.accept(row)));
recordConsumer.close();
}

protected static Path buildTablePath(final Path storageRoot, final String tableName) {
return storageRoot
.resolve(DB_FOLDER_NAME)
.resolve(String.format("%s.yaml", tableName.toUpperCase()));
}

private void dumpConfigsDatabase(final Path parentFolder) throws IOException {
for (final Map.Entry<String, Stream<JsonNode>> configEntry : configRepository.dumpConfigs().entrySet()) {
writeConfigsToArchive(parentFolder, configEntry.getKey(), configEntry.getValue());
Expand Down
54 changes: 0 additions & 54 deletions airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@
import io.airbyte.api.model.UploadRead;
import io.airbyte.commons.enums.Enums;
import io.airbyte.commons.io.Archives;
import io.airbyte.commons.io.IOs;
import io.airbyte.commons.json.Jsons;
import io.airbyte.commons.stream.MoreStreams;
import io.airbyte.commons.version.AirbyteVersion;
import io.airbyte.commons.yaml.Yamls;
import io.airbyte.config.AirbyteConfig;
Expand All @@ -28,7 +26,6 @@
import io.airbyte.config.persistence.ConfigNotFoundException;
import io.airbyte.config.persistence.ConfigPersistence;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.db.instance.jobs.JobsDatabaseSchema;
import io.airbyte.scheduler.persistence.DefaultJobPersistence;
import io.airbyte.scheduler.persistence.JobPersistence;
import io.airbyte.scheduler.persistence.WorkspaceHelper;
Expand Down Expand Up @@ -129,9 +126,6 @@ public void importDataWithSeed(final AirbyteVersion targetVersion, final File ar
throw e;
}

// 3. Import Postgres content
importDatabaseFromArchive(sourceRoot, targetVersion);

// 4. Import Configs and update connector definitions
importConfigsFromArchive(sourceRoot, false);
configRepository.loadData(seedPersistence);
Expand Down Expand Up @@ -226,28 +220,6 @@ protected static Path buildConfigPath(final Path storageRoot, final ConfigSchema
.resolve(String.format("%s.yaml", schemaType.name()));
}

// Postgres Portion
public void importDatabaseFromArchive(final Path storageRoot, final AirbyteVersion airbyteVersion) throws IOException {
try {
final Map<JobsDatabaseSchema, Stream<JsonNode>> data = new HashMap<>();
for (final JobsDatabaseSchema tableType : JobsDatabaseSchema.values()) {
final Path tablePath = buildTablePath(storageRoot, tableType.name());
Stream<JsonNode> tableStream = readTableFromArchive(tableType, tablePath);

if (tableType == JobsDatabaseSchema.AIRBYTE_METADATA) {
tableStream = replaceDeploymentMetadata(jobPersistence, tableStream);
}

data.put(tableType, tableStream);
}
jobPersistence.importDatabase(airbyteVersion.serialize(), data);
LOGGER.info("Successful upgrade of airbyte postgres database from archive");
} catch (final Exception e) {
LOGGER.warn("Postgres database version upgrade failed, reverting to state previous to migration.");
throw e;
}
}

/**
* The deployment concept is specific to the environment that Airbyte is running in (not the data
* being imported). Thus, if there is a deployment in the imported data, we filter it out. In
Expand Down Expand Up @@ -278,32 +250,6 @@ static Stream<JsonNode> replaceDeploymentMetadata(final JobPersistence postgresP
return stream;
}

protected static Path buildTablePath(final Path storageRoot, final String tableName) {
return storageRoot
.resolve(DB_FOLDER_NAME)
.resolve(String.format("%s.yaml", tableName.toUpperCase()));
}

private Stream<JsonNode> readTableFromArchive(final JobsDatabaseSchema tableSchema,
final Path tablePath)
throws FileNotFoundException {
final JsonNode schema = tableSchema.getTableDefinition();
if (schema != null) {
return MoreStreams.toStream(Yamls.deserialize(IOs.readFile(tablePath)).elements())
.peek(r -> {
try {
jsonSchemaValidator.ensure(schema, r);
} catch (final JsonValidationException e) {
throw new IllegalArgumentException(
"Archived Data Schema does not match current Airbyte Data Schemas", e);
}
});
} else {
throw new FileNotFoundException(String
.format("Airbyte Database table %s was not found in the archive", tableSchema.name()));
}
}

private void checkDBVersion(final AirbyteVersion airbyteVersion) throws IOException {
final Optional<AirbyteVersion> airbyteDatabaseVersion = jobPersistence.getVersion().map(AirbyteVersion::new);
airbyteDatabaseVersion
Expand Down

0 comments on commit ffbf8bc

Please sign in to comment.