What: Allow proto definition providers to specify the root of the pro…

…to directory hierarchy.

Why: To allow greater flexibility in source code organization.

README.md

@@ -65,6 +65,13 @@ This will generate a manifest file named `protop.json`.
 }
 ```
 
+If the root of the proto files are a subdirectory (eg `src/main/proto`), add the following to `protop.json`:
+```json
+  "root-dir": "src/main/proto"
+```
+
+This will allow clients not to have to adjust the protoc include path.
+
 ## Publish ("link") locally
 This will make the project accessible to all other projects that run `sync` with links enabled.
 ```bash
@@ -81,12 +88,12 @@ To unlink all projects system-wide:
 $ protop links clean
 ```
 
-### Publish to a repository*
+### Publish to a repository
 ```bash
 $ protop publish -r=https://repository.example.com
 ```
 
-*There is an implementation of a Nexus plugin for protop required for this to work. More details [here](https://github.com/protop-io/nexus-repository-protop). Coming soon, there will be better documentation on the API of the repository itself.
+> There is an implementation of a Nexus plugin for protop required for this to work. More details [here](https://github.com/protop-io/nexus-repository-protop). Coming soon, there will be better documentation on the API of the repository itself.
 
 ### Sync local/external dependencies
 Run the following with `-l` or `--use-links` to include local/linked dependencies, or run without it to only include dependencies from the network.
@@ -120,35 +127,7 @@ $ protop cache clean
 ## Use with Gradle or other build tools
 
 ### Use with Gradle
-There isn't a custom Gradle plugin for protop (yet). Even so, the implementation is quite simple. Assuming you have an existing `build.gradle` setup for a protobuf project, add the following task to the root project:
-```groovy
-task protop(type: Exec) {
-    workingDir "."
-    commandLine "protop", "sync"
-}
-```
-This task will simply run `protop sync`. To invoke it upon `gradle build` and ensure that it is run before the protos are generated, alter the `protobuf` block (or add it now):
-```groovy
-protobuf {
-    // ...
-    generateProtoTasks {
-        // ...
-        all().each { task -> task.dependsOn protop }
-    }
-}
-```
-
-Finally, make sure the compiler will find all the synced protos:
-```groovy
-sourceSets {
-    main {
-        // ...
-        proto {
-            srcDir ".protop/path"
-        }
-    }
-}
-```
+Use the [Gradle plugin](https://github.com/google/protobuf-gradle-plugin). 
 
 ### Use with protoc directly
 With dependencies already synced, you can call `protoc` in a project that looks like the one above:

build.gradle

@@ -23,19 +23,24 @@ allprojects {
 
 subprojects {
     dependencies {
+        annotationProcessor "org.projectlombok:lombok:$lombokVersion"
+
+        compileOnly "org.projectlombok:lombok:$lombokVersion"
+
+        implementation "ch.qos.logback:logback-classic:1.2.3"
         implementation "com.google.guava:guava:27.0.1-jre"
         implementation "com.fasterxml.jackson.core:jackson-databind:2.9.9.3"
         implementation "com.fasterxml.jackson.core:jackson-annotations:2.9.9"
-        implementation "javax.validation:validation-api:2.0.1.Final"
         implementation "commons-io:commons-io:2.6"
-        implementation "ch.qos.logback:logback-classic:1.2.3"
         implementation "io.reactivex.rxjava2:rxjava:2.2.17"
-
-        compileOnly "org.projectlombok:lombok:$lombokVersion"
+        implementation "javax.validation:validation-api:2.0.1.Final"
 
-        annotationProcessor "org.projectlombok:lombok:$lombokVersion"
+        testImplementation "org.assertj:assertj-core:3.+"
+        testImplementation "org.junit.jupiter:junit-jupiter-engine:5.+"
+    }
 
-        testImplementation "junit:junit:4.12"
+    test {
+        useJUnitPlatform()
     }
 
     tasks.withType(org.gradle.api.tasks.javadoc.Javadoc).all { enabled = false }

protop-core/src/main/java/io/protop/core/manifest/Manifest.java

@@ -23,7 +23,7 @@
 @SuperBuilder
 @JsonInclude(JsonInclude.Include.NON_NULL)
 @JsonIgnoreProperties(ignoreUnknown = true)
-@JsonPropertyOrder({"name", "org", "version", "description", "keywords", "license", "homepage"})
+@JsonPropertyOrder({"name", "org", "version", "description", "root-dir", "keywords", "license", "homepage"})
 public class Manifest {
 
     private static final Logger logger = Logger.getLogger(Manifest.class);
@@ -42,6 +42,9 @@ public class Manifest {
 //    @JsonSerialize(converter = PathListToStringList.class)
 //    private List<Path> include;
 
+    @JsonProperty("root-dir")
+    private Path rootDir;
+
     @JsonInclude(JsonInclude.Include.NON_NULL)
     @JsonProperty("dependencies")
     private DependencyMap dependencies;
@@ -67,6 +70,7 @@ public class Manifest {
             @JsonProperty("version") @NotNull Version version,
             @JsonProperty("organization") @NotNull String organization,
 //            @JsonProperty("include") List<Path> include,
+            @JsonProperty("root-dir") @NotNull Path rootDir,
             @JsonProperty("dependencies") @JsonDeserialize(converter = DependencyMapDeserializer.class)
                     DependencyMap dependencies,
             @JsonProperty("description") String description,
@@ -78,6 +82,7 @@ public class Manifest {
         this.version = version;
         this.organization = organization;
 //        this.include = Objects.isNull(include) ? ImmutableList.of() : ImmutableList.copyOf(include);
+        this.rootDir = rootDir;
         this.dependencies = dependencies;
         this.description = description;
         this.readme = readme;
@@ -106,4 +111,8 @@ public static Optional<Manifest> from(Path directory) {
             throw new RuntimeException(message, e);
         }
     }
+
+    public static Optional<Manifest> from(final File directory) {
+        return from(directory.toPath());
+    }
 }

protop-core/src/main/java/io/protop/core/sync/FileWithRootDir.java

@@ -0,0 +1,37 @@
+package io.protop.core.sync;
+
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+@EqualsAndHashCode
+@Getter
+class FileWithRootDir {
+    private final Path rootDir;
+    private final Path target;
+
+    public FileWithRootDir(final Path rootDir, final File file) {
+        this(rootDir, file.toPath());
+    }
+
+    public FileWithRootDir(final Path rootDir, final Path target) {
+        this.rootDir = rootDir != null ? rootDir : Path.of("");
+        this.target = target;
+    }
+
+    public void createSymbolicLink(final Path protopPathDir, final Path linkWithRootDir) throws IOException {
+        final Path linkRelative = protopPathDir
+                .resolve(rootDir)
+                .relativize(linkWithRootDir);
+        final Path linkWithoutRootDir = protopPathDir.resolve(linkRelative);
+
+        Files.createDirectories(linkWithoutRootDir.getParent());
+        Files.createSymbolicLink(
+                linkWithoutRootDir,
+                target);
+    }
+}

protop-core/src/main/java/io/protop/core/sync/SyncService.java

@@ -1,28 +1,36 @@
 package io.protop.core.sync;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import io.protop.core.Context;
 import io.protop.core.auth.AuthService;
 import io.protop.core.cache.CacheService;
 import io.protop.core.logs.Logger;
 import io.protop.core.manifest.Coordinate;
 import io.protop.core.manifest.DependencyMap;
+import io.protop.core.manifest.Manifest;
 import io.protop.core.manifest.revision.RevisionSource;
 import io.protop.core.storage.Storage;
 import io.protop.core.storage.StorageService;
 import io.protop.core.sync.status.SyncStatus;
 import io.protop.core.sync.status.Syncing;
 import io.reactivex.Observable;
+import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.RequiredArgsConstructor;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.TrueFileFilter;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.FileAlreadyExistsException;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
@@ -97,20 +105,34 @@ private List<File> getProtoFilesInDir(Path dir) {
                 .collect(Collectors.toCollection(ArrayList::new));
     }
 
+    @VisibleForTesting
+    @EqualsAndHashCode
     @Getter
-    private static class DirectoryWithFiles {
-        private final Map<String, File> files = new HashMap<>();
-        private final Map<String, DirectoryWithFiles> subdirectories = new HashMap<>();
+    static class DirectoryWithFiles {
+        private final Map<String, FileWithRootDir> files;
+        private final Map<String, DirectoryWithFiles> subdirectories;
+
+        DirectoryWithFiles() {
+            this(new HashMap<>(), new HashMap<>());
+        }
+
+        @VisibleForTesting
+        DirectoryWithFiles(final Map<String, FileWithRootDir> files, final Map<String, DirectoryWithFiles> subdirectories) {
+            this.files = files;
+            this.subdirectories = subdirectories;
+        }
     }
 
     private List<File> validProtoFilesOrDirectories(File[] files) {
         List<String> invalidDirectoryNames = ImmutableList.of("node_modules");
         List<File> validFiles = new ArrayList<>();
         for (File file : files) {
-            if (!file.getName().startsWith(".") && !file.isHidden()) {
-                if (file.isDirectory() && !invalidDirectoryNames.contains(file.getName())) {
+            final String fileName = file.getName();
+
+            if (!fileName.startsWith(".") && !file.isHidden()) {
+                if (file.isDirectory() && !invalidDirectoryNames.contains(fileName)) {
                     validFiles.add(file);
-                } else if (file.isFile() && file.getName().endsWith(".proto")) {
+                } else if (file.isFile() && fileName.endsWith(".proto")) {
                     validFiles.add(file);
                 }
             }
@@ -121,16 +143,32 @@ private List<File> validProtoFilesOrDirectories(File[] files) {
     /**
      * Filter the children directories and files and add to the given tree.
      */
-    private void filterValidChildren(DirectoryWithFiles directoryWithFiles, File path) throws FileAlreadyExistsException {
+    private void filterValidChildren(
+            final DirectoryWithFiles directoryWithFiles,
+            final File project) throws FileAlreadyExistsException {
+        final Path rootDir = Manifest
+                .from(project)
+                .map(Manifest::getRootDir)
+                .orElse(Path.of(""));
+
+        filterValidChildren(directoryWithFiles, project, rootDir);
+    }
+
+    @VisibleForTesting
+    void filterValidChildren(
+            final DirectoryWithFiles directoryWithFiles,
+            final File path,
+            final Path rootDir) throws FileAlreadyExistsException {
         if (path.isDirectory()) {
-            File[] children = Optional.ofNullable(path.listFiles())
+            final File[] children = Optional.ofNullable(path.listFiles())
                     .orElse(new File[]{});
+
             for (File child : validProtoFilesOrDirectories(children)) {
                 if (child.isDirectory()) {
                     DirectoryWithFiles childDirectory = directoryWithFiles.subdirectories.computeIfAbsent(
                             child.getName(),
                             name -> new DirectoryWithFiles());
-                    filterValidChildren(childDirectory, child);
+                    filterValidChildren(childDirectory, child, rootDir);
                     // If the child directory ended up with nothing in it, remove it.
                     // This isn't the most efficient thing, but it does save us from having to traverse the
                     // entire tree again in order to remove empty directories.
@@ -145,7 +183,7 @@ private void filterValidChildren(DirectoryWithFiles directoryWithFiles, File pat
                                 path.getName());
                         throw new FileAlreadyExistsException(message);
                     } else {
-                        directoryWithFiles.files.put(child.getName(), child);
+                        directoryWithFiles.files.put(child.getName(), new FileWithRootDir(rootDir, child));
                     }
                 }
             }
@@ -157,25 +195,33 @@ private void filterValidChildren(DirectoryWithFiles directoryWithFiles, File pat
     /**
      * Create subdirectories and symlink files in the given parent.
      */
-    private void mergeChildrenToParentDirectory(File parent, DirectoryWithFiles children) throws IOException {
-        for (Map.Entry<String, File> fileEntry : children.getFiles().entrySet()) {
-            Files.createSymbolicLink(parent.toPath().resolve(fileEntry.getKey()),
-                    fileEntry.getValue().toPath());
+    private void mergeChildrenToParentDirectory(
+            final Path protopPathDir, final Path parent, final DirectoryWithFiles children) throws IOException {
+        for (Map.Entry<String, FileWithRootDir> fileEntry : children.getFiles().entrySet()) {
+            final Path link = parent.resolve(fileEntry.getKey());
+
+            fileEntry.getValue().createSymbolicLink(protopPathDir, link);
         }
+
         for (Map.Entry<String, DirectoryWithFiles> directoryEntry : children.getSubdirectories().entrySet()) {
-            Path created = Files.createDirectory(parent.toPath().resolve(directoryEntry.getKey()));
-            mergeChildrenToParentDirectory(created.toFile(), directoryEntry.getValue());
+            final Path subdirectory = parent.resolve(directoryEntry.getKey());
+
+            mergeChildrenToParentDirectory(protopPathDir, subdirectory, directoryEntry.getValue());
         }
     }
 
-    private void mergeDepsToPath(Path depsDir) throws IOException {
-        DirectoryWithFiles depsTree = new DirectoryWithFiles();
-        File[] orgs = Optional.ofNullable(depsDir.toFile().listFiles())
+    private void mergeDepsToPath(final Path depsDir) throws IOException {
+        final DirectoryWithFiles depsTree = new DirectoryWithFiles();
+        final File[] orgs = Optional
+                .ofNullable(depsDir.toFile().listFiles())
                 .orElse(new File[]{});
+
         for (File org : orgs) {
             if (org.isDirectory()) {
-                File[] projects = Optional.ofNullable(org.listFiles())
+                final File[] projects = Optional
+                        .ofNullable(org.listFiles())
                         .orElse(new File[]{});
+
                 for (File project : projects) {
                     // For every project, traverse its items and add to the dependency tree.
                     if (project.isDirectory()) {
@@ -185,8 +231,8 @@ private void mergeDepsToPath(Path depsDir) throws IOException {
             }
         }
 
-        Path mergedDepsPath = resolveEmptySubDir(Storage.ProjectDirectory.PATH);
-        mergeChildrenToParentDirectory(mergedDepsPath.toFile(), depsTree);
+        final Path mergedDepsPath = resolveEmptySubDir(Storage.ProjectDirectory.PATH);
+        mergeChildrenToParentDirectory(mergedDepsPath, mergedDepsPath, depsTree);
     }
 
     /**