OAK-9866 + OAK-9872 (apache#641)

* OAK-9866 Add command line argument for append mode in oak-run segment/copy * OAK-9872 Align oak-run segment-copy flat flag implementation with documentation Co-authored-by: Lucas Weitzendorf <[email protected]>
mrzhang638993 · Jul 29, 2022 · 92dffbd · 92dffbd
1 parent e71d4f1
commit 92dffbd
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 13 deletions.
diff --git a/oak-doc/src/site/markdown/nodestore/segment/overview.md b/oak-doc/src/site/markdown/nodestore/segment/overview.md
@@ -678,7 +678,7 @@ Besides the local storage in TAR files (previously known as TarMK), support for
 
 ### <a name="segment-copy"/> Segment-Copy
 ```
-java -jar oak-run.jar segment-copy SOURCE DESTINATION [--last <REV_COUNT>] [--flat] [--max-size-gb <MAX_SIZE_GB>]
+java -jar oak-run.jar segment-copy SOURCE DESTINATION [--last <REV_COUNT>] [--flat] [--append] [--max-size-gb <MAX_SIZE_GB>]
 ```
 
 The `segment-copy` command allows the "translation" of the Segment Store at `SOURCE` from one persistence type (e.g. local TarMK Segment Store) to a different persistence type (e.g. remote Azure or AWS Segment Store), saving the resulted Segment Store at `DESTINATION`. 
@@ -693,7 +693,9 @@ Please refer to the [Remote Segment Stores](#remote-segment-stores) section for
 
 The optional `--last [Integer]` argument can be used to control the maximum number of revisions to be copied from the journal (default is 1).
 
-The optional `--flat [Boolean]` argument can be specified for allowing the copy process to write the segments at `DESTINATION` in a flat hierarchy, that is without writing them in tar archives. 
+The optional `--flat` argument can be specified for allowing the copy process to write the segments at `DESTINATION` in a flat hierarchy, that is without writing them in tar archives. 
+
+The optional `--append` argument can be specified for running segment copy in append mode. This causes existing segments in `DESTINATION` to be skipped instead of overwritten.
 
 The optional `--max-size-gb <MAX_SIZE_GB>` argument can be used for specifying to copy up to `MAX_SIZE_GB` segments from `SOURCE`.
 

diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentCopyCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentCopyCommand.java
@@ -38,9 +38,9 @@ public void execute(String... args) throws Exception {
                 .withOptionalArg()
                 .ofType(Integer.class);
 
-        OptionSpec<Boolean> flat = parser.accepts("flat", "copy segments in flat hierarchy")
-                .withOptionalArg()
-                .ofType(Boolean.class);
+        OptionSpec<Void> flat = parser.accepts("flat", "copy segments in flat hierarchy");
+
+        OptionSpec<Void> append = parser.accepts("append", "skip existing segments");
 
         OptionSpec<Integer> maxSizeGb = parser.accepts("max-size-gb", "define maximum size of archives to be copied")
                 .withOptionalArg()
@@ -75,15 +75,16 @@ public void execute(String... args) throws Exception {
                     .withSource(source)
                     .withDestination(destination)
                     .withOutWriter(out)
-                    .withErrWriter(err);
+                    .withErrWriter(err)
+                    .withAppendMode(options.has(append));
 
             if (options.has(last)) {
                 builder.withRevisionsCount(last.value(options) != null ? last.value(options) : 1);
             }
 
-            if (options.has(flat) && options.has(maxSizeGb)) {
+            if (options.has(maxSizeGb)) {
                 builder.withMaxSizeGb(maxSizeGb.value(options));
-                builder.withFlat(flat.value(options));
+                builder.withFlat(options.has(flat));
             }
 
             System.exit(builder.build().run());

diff --git a/...segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java b/...segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java
@@ -91,6 +91,8 @@ public static class Builder {
 
         private Boolean flat = false;
 
+        private Boolean appendMode = false;
+
         private Integer maxSizeGb = 1;
 
         private Builder() {
@@ -186,14 +188,26 @@ public Builder withRevisionsCount(Integer revisionsCount) {
          * If enabled, the segments hierarchy will be copied without any
          * TAR archive being created, in a flat hierarchy.
          *
-         * @param flat flag controlling the copying in flat hierarchy
+         * @param flat flag controlling the copying in flat hierarchy.
          * @return this builder.
          */
         public Builder withFlat(Boolean flat) {
             this.flat = flat;
             return this;
         }
 
+        /**
+         * If enabled, existing segments will be skipped instead of
+         * overwritten in the copy process.
+         *
+         * @param appendMode flag controlling the segment write behavior.
+         * @return this builder.
+         */
+        public Builder withAppendMode(Boolean appendMode) {
+            this.appendMode = appendMode;
+            return this;
+        }
+
         /**
          * Parameter for configuring the maximum size of the segment store transfer
          *
@@ -234,6 +248,8 @@ public SegmentCopy build() {
 
     private final Boolean flat;
 
+    private final Boolean appendMode;
+
     private final Integer maxSizeGb;
 
     private SegmentNodeStorePersistence srcPersistence;
@@ -249,6 +265,7 @@ public SegmentCopy(Builder builder) {
         this.destPersistence = builder.destPersistence;
         this.revisionCount = builder.revisionsCount;
         this.flat = builder.flat;
+        this.appendMode = builder.appendMode;
         this.maxSizeGb = builder.maxSizeGb;
         this.outWriter = builder.outWriter;
         this.errWriter = builder.errWriter;
@@ -355,12 +372,16 @@ public int run() {
                 printMessage(outWriter, "Source: {0}", srcDescription);
                 printMessage(outWriter, "Destination: {0}", destDescription);
 
-                SegmentStoreMigrator migrator = new SegmentStoreMigrator.Builder()
+                SegmentStoreMigrator.Builder migratorBuilder = new SegmentStoreMigrator.Builder()
                         .withSourcePersistence(srcPersistence, srcDescription)
-                        .withTargetPersistence(destPersistence, destDescription).withRevisionCount(revisionCount)
-                        .build();
+                        .withTargetPersistence(destPersistence, destDescription)
+                        .withRevisionCount(revisionCount);
+
+                if (appendMode) {
+                    migratorBuilder.setAppendMode();
+                }
 
-                migrator.migrate();
+                migratorBuilder.build().migrate();
 
             } catch (Exception e) {
                 watch.stop();