Skip to content

Commit

Permalink
Refactor to ensure that resources are cleaned up
Browse files Browse the repository at this point in the history
  • Loading branch information
@Calvin-L
Calvin-L committed Oct 5, 2023
1 parent 5b61a5b commit 193fa9b
Showing 1 changed file with 130 additions and 102 deletions.
232 changes: 130 additions & 102 deletions src/main/java/cal/bkup/Main.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,104 @@ private static void showHelp(Options options) {
}

public static void main(String[] args) throws IOException {
var flags = parseCLIFlags(args);

if (flags == null) {
return;
}

// ------------------------------------------------------------------------------
// Set up actors and configuration

final Config config;
try {
config = loadConfig(CFG_FILE);
} catch (FileNotFoundException e) {
System.err.println("Config file '" + CFG_FILE + "' not found");
System.exit(1);
return;
}

final BlobTransformer transform = new XZCompression();
final IndexFormat indexFormat = new VersionedIndexFormat(
new JsonIndexFormatV01(),
new JsonIndexFormatV02(),
new JsonIndexFormatV03(),
new JsonIndexFormatV04(),
new JsonIndexFormatV05());
final BackerUpper backupper;
final UnreliableWallClock clock = UnreliableWallClock.SYSTEM_CLOCK;

if (flags.local) {
var registerLocation = Paths.get("/tmp/backup/register.db");
try (var register = new SQLiteStringRegister(registerLocation)) {
EventuallyConsistentDirectory dir = new LocalDirectory(Paths.get("/tmp/backup/indexes"));
ConsistentBlob indexStore = new ConsistentBlobOnEventuallyConsistentDirectory(register, dir);
EventuallyConsistentBlobStore blobStore = new BlobStoreOnDirectory(new LocalDirectory(Paths.get("/tmp/backup/blobs")));
backupper = new BackerUpper(
indexStore, indexFormat,
blobStore, transform,
clock);
run(backupper, config, flags);
} catch (SQLException e) {
System.err.println("Failed to create SQLiteStringRegister at " + registerLocation);
e.printStackTrace();
System.exit(1);
}
} else {
try (var credentials = AWSTools.credentialsProvider();
var dynamoClient = DynamoDbClient.builder()
.credentialsProvider(credentials)
.region(AWS_REGION)
.build();
var s3Client = S3Client.builder()
.credentialsProvider(credentials)
.region(AWS_REGION)
.build();
var glacierClient = GlacierClient.builder()
.credentialsProvider(credentials)
.region(AWS_REGION)
.build()) {

StringRegister register = new DynamoDBStringRegister(
dynamoClient,
DYNAMO_TABLE,
DYNAMO_REGISTER);

EventuallyConsistentDirectory dir = new S3Directory(
s3Client,
S3_BUCKET);

ConsistentBlob indexStore = new ConsistentBlobOnEventuallyConsistentDirectory(register, dir);

EventuallyConsistentBlobStore blobStore = new GlacierBlobStore(
glacierClient,
GLACIER_VAULT_NAME);

backupper = new BackerUpper(
indexStore, indexFormat,
blobStore, transform,
clock);

run(backupper, config, flags);
}
}
}

private static class WhatToDo {
boolean dryRun;
boolean list;
boolean gc;
boolean backup;
boolean dumpIndex;
boolean local;
int numToCheck;
boolean test;
String password = "";
String newPassword = "";
}

private static @Nullable WhatToDo parseCLIFlags(String... args) {
Options options = new Options();

// flags
Expand All @@ -157,26 +255,27 @@ public static void main(String[] args) throws IOException {
System.err.println("Failed to parse options: " + e);
showHelp(options);
System.exit(1);
return;
return null;
}

if (cli.hasOption('h')) {
showHelp(options);
return;
return null;
}

final boolean dryRun = cli.hasOption('d');
final boolean list = cli.hasOption('l');
final boolean gc = cli.hasOption("gc");
final boolean backup = cli.hasOption("backup");
final boolean dumpIndex = cli.hasOption("dump-index");
final boolean local = cli.hasOption("local");
final int numToCheck = cli.hasOption('c') ? Integer.parseInt(cli.getOptionValue('c')) : 0;
final boolean test = cli.hasOption('t');

if (!backup && !list && numToCheck == 0 && !gc && !dumpIndex && !test) {
WhatToDo result = new WhatToDo();
result.dryRun = cli.hasOption('d');
result.list = cli.hasOption('l');
result.gc = cli.hasOption("gc");
result.backup = cli.hasOption("backup");
result.dumpIndex = cli.hasOption("dump-index");
result.local = cli.hasOption("local");
result.numToCheck = cli.hasOption('c') ? Integer.parseInt(cli.getOptionValue('c')) : 0;
result.test = cli.hasOption('t');

if (!result.backup && !result.list && result.numToCheck == 0 && !result.gc && !result.dumpIndex && !result.test) {
System.err.println("No action specified. Did you mean to pass '-b'?");
return;
return null;
}

final String password = cli.hasOption('p') ? cli.getOptionValue('p') : Util.readPassword("Password");
Expand All @@ -195,94 +294,24 @@ public static void main(String[] args) throws IOException {
if (password == null || newPassword == null) {
System.err.println("No password; refusing to proceed");
System.exit(1);
return null;
}

if (!Objects.equals(password, newPassword) && !backup) {
if (!Objects.equals(password, newPassword) && !result.backup) {
System.err.println("WARNING: a new password was specified without '-b'; the new password will be ignored.");
}

// ------------------------------------------------------------------------------
// Set up actors and configuration
result.password = password;
result.newPassword = newPassword;

final Config config;
try {
config = loadConfig(CFG_FILE);
} catch (FileNotFoundException e) {
System.err.println("Config file '" + CFG_FILE + "' not found");
System.exit(1);
return;
}

final BlobTransformer transform = new XZCompression();
final IndexFormat indexFormat = new VersionedIndexFormat(
new JsonIndexFormatV01(),
new JsonIndexFormatV02(),
new JsonIndexFormatV03(),
new JsonIndexFormatV04(),
new JsonIndexFormatV05());
final BackerUpper backupper;
final UnreliableWallClock clock = UnreliableWallClock.SYSTEM_CLOCK;

if (local) {
var registerLocation = Paths.get("/tmp/backup/register.db");
StringRegister register = null;
try {
@SuppressWarnings("required.method.not.called") // TODO
var r = new SQLiteStringRegister(registerLocation);
register = r;
} catch (SQLException e) {
System.err.println("Failed to create SQLiteStringRegister at " + registerLocation);
e.printStackTrace();
System.exit(1);
}
EventuallyConsistentDirectory dir = new LocalDirectory(Paths.get("/tmp/backup/indexes"));
ConsistentBlob indexStore = new ConsistentBlobOnEventuallyConsistentDirectory(register, dir);
EventuallyConsistentBlobStore blobStore = new BlobStoreOnDirectory(new LocalDirectory(Paths.get("/tmp/backup/blobs")));
backupper = new BackerUpper(
indexStore, indexFormat,
blobStore, transform,
clock);
} else {
@SuppressWarnings("required.method.not.called") // TODO
var credentials = AWSTools.credentialsProvider();

@SuppressWarnings("required.method.not.called") // TODO
StringRegister register = new DynamoDBStringRegister(
DynamoDbClient.builder()
.credentialsProvider(credentials)
.region(AWS_REGION)
.build(),
DYNAMO_TABLE,
DYNAMO_REGISTER);

@SuppressWarnings("required.method.not.called") // TODO
EventuallyConsistentDirectory dir = new S3Directory(
S3Client.builder()
.credentialsProvider(credentials)
.region(AWS_REGION)
.build(),
S3_BUCKET);

ConsistentBlob indexStore = new ConsistentBlobOnEventuallyConsistentDirectory(register, dir);

@SuppressWarnings("required.method.not.called") // TODO
EventuallyConsistentBlobStore blobStore = new GlacierBlobStore(
GlacierClient.builder()
.credentialsProvider(credentials)
.region(AWS_REGION)
.build(),
GLACIER_VAULT_NAME);

backupper = new BackerUpper(
indexStore, indexFormat,
blobStore, transform,
clock);
}
return result;
}

// ------------------------------------------------------------------------------
// Do the work
private static void run(BackerUpper backupper, Config config, WhatToDo flags) throws IOException {
String password = flags.password;
String newPassword = flags.newPassword;

if (test) {
if (flags.test) {
try (var credentials = AWSTools.credentialsProvider();
var client = DynamoDbClient.builder()
.credentialsProvider(credentials)
Expand All @@ -304,7 +333,7 @@ public static void main(String[] args) throws IOException {
System.out.println("Test OK");
}

if (backup) {
if (flags.backup) {
System.out.println("Scanning filesystem...");
List<SymLink> symlinks = new ArrayList<>();
List<HardLink> hardlinks = new ArrayList<>();
Expand All @@ -317,7 +346,7 @@ public static void main(String[] args) throws IOException {
System.out.println(" uploaded bytes: " + Util.formatSize(plan.estimatedBytesUploaded()));
System.out.println(" backup cost now: " + plan.estimatedExecutionCost());
System.out.println(" monthly maintenance: " + plan.estimatedMonthlyCost());
if (!dryRun && confirm("Proceed?")) {
if (!flags.dryRun && confirm("Proceed?")) {
try {
plan.execute(fs);
} catch (BackupIndex.MergeConflict mergeConflict) {
Expand All @@ -329,21 +358,21 @@ public static void main(String[] args) throws IOException {
}
}

if (list) {
if (flags.list) {
backupper.list(newPassword).forEach(info -> {
System.out.println('[' + info.system().toString() + "] " + info.latestRevision() + ": " + info.path());
});
}

if (gc) {
if (flags.gc) {
System.out.println("Planning cleanup...");
BackerUpper.CleanupPlan plan = backupper.planCleanup(password, Duration.ofDays(60), COST_MODEL);
System.out.println("Estimated costs:");
System.out.println(" deleted blobs: " + plan.totalBlobsReclaimed() + " (" + plan.untrackedBlobsReclaimed() + " of which are not known to the index)");
System.out.println(" reclaimed bytes: " + Util.formatSize(plan.bytesReclaimed()));
System.out.println(" backup cost now: " + plan.estimatedExecutionCost());
System.out.println(" monthly maintenance: " + plan.estimatedMonthlyCost());
if (!dryRun && confirm("Proceed?")) {
if (!flags.dryRun && confirm("Proceed?")) {
try {
plan.execute();
} catch (BackupIndex.MergeConflict mergeConflict) {
Expand All @@ -353,7 +382,7 @@ public static void main(String[] args) throws IOException {
}
}

if (dumpIndex) {
if (flags.dumpIndex) {
try (InputStream in = new BufferedInputStream(backupper.readRawIndex(newPassword))) {
Util.copyStream(in, System.out);
} catch (NoValue noValue) {
Expand All @@ -364,15 +393,15 @@ public static void main(String[] args) throws IOException {
}
}

if (numToCheck > 0) {
if (flags.numToCheck > 0) {
List<Pair<Path, Sha256AndSize>> candidates = backupper.list(newPassword)
.filter(item -> item.system().equals(config.systemName()))
.filter(item -> item.latestRevision() instanceof BackupIndex.RegularFileRev)
.map(item -> new Pair<>(item.path(), ((BackupIndex.RegularFileRev) item.latestRevision()).summary()))
.collect(Collectors.toList());
Random random = new Random();
Collections.shuffle(candidates, random);
int len = Math.min(numToCheck, candidates.size());
int len = Math.min(flags.numToCheck, candidates.size());
List<Sha256AndSize> localSummaries = new ArrayList<>();
List<Sha256AndSize> remoteSummaries = new ArrayList<>();
try (ProgressDisplay display = new ProgressDisplay(len)) {
Expand Down Expand Up @@ -405,7 +434,6 @@ public static void main(String[] args) throws IOException {
System.exit(1);
}
}

}

private static boolean confirm(String prompt) {
Expand Down

0 comments on commit 193fa9b

Please sign in to comment.