diff --git a/README.adoc b/README.adoc index 5131f55733e..85e0362cd8a 100644 --- a/README.adoc +++ b/README.adoc @@ -75,18 +75,24 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 4.3.0 - 4.3.18 |Neo4j 4.4.0 - 4.4.11 -.4+<.^|GDS 2.2.x +.6+<.^|GDS 2.2.x |Neo4j 4.3.15 - 4.3.23 -.7+.^|Java 11 & Java 17 +.15+.^|Java 11 & Java 17 |Neo4j 4.4.9 - 4.4.16 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 -.3+<.^|GDS 2.3.x -|Neo4j 4.4.9 - 4.4.16 +|Neo4j 5.4.0 +.9+<.^|GDS 2.3.x +|Neo4j 4.4.9 - 4.4.22 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 +|Neo4j 5.4.0 +|Neo4j 5.5.0 +|Neo4j 5.6.0 +|Neo4j 5.7.0 +|Neo4j 5.8.0 |=== NOTE: Preview releases are not automatically made available in Neo4j Desktop. They need to be installed manually. @@ -130,7 +136,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.2.6 + 2.3.6 ---- @@ -142,21 +148,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.2.6 + 2.3.8 org.neo4j.gds algo - 2.2.6 + 2.3.8 org.neo4j.gds alpha-algo - 2.2.6 + 2.3.8 ---- @@ -168,28 +174,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.2.6 + 2.3.8 org.neo4j.gds proc - 2.2.6 + 2.3.8 org.neo4j.gds alpha-proc - 2.2.6 + 2.3.8 org.neo4j.gds write-services - 2.2.6 + 2.3.8 ---- diff --git a/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java b/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java index 525e868a618..9b1262ce702 100644 --- a/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java +++ b/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds; +import org.jetbrains.annotations.NotNull; import org.neo4j.gds.config.AlgoBaseConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.utils.mem.MemoryEstimation; @@ -26,6 +27,7 @@ import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.TaskTreeProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory; import org.neo4j.gds.core.utils.warnings.UserLogRegistryFactory; @@ -55,18 +57,47 @@ default ALGO build( TaskRegistryFactory taskRegistryFactory, UserLogRegistryFactory userLogRegistryFactory ) { - var progressTask = progressTask(graphOrGraphStore, configuration); - var progressTracker = new TaskProgressTracker( - progressTask, + var progressTracker = createProgressTracker( + configuration, log, - configuration.concurrency(), - configuration.jobId(), taskRegistryFactory, - userLogRegistryFactory + userLogRegistryFactory, + progressTask(graphOrGraphStore, configuration) ); return build(graphOrGraphStore, configuration, progressTracker); } + @NotNull + private ProgressTracker createProgressTracker( + CONFIG configuration, + Log log, + TaskRegistryFactory taskRegistryFactory, + UserLogRegistryFactory userLogRegistryFactory, + Task progressTask + ) { + ProgressTracker progressTracker; + if (configuration.logProgress()) { + progressTracker = new TaskProgressTracker( + progressTask, + log, + configuration.concurrency(), + configuration.jobId(), + taskRegistryFactory, + userLogRegistryFactory + ); + } else { + progressTracker = new TaskTreeProgressTracker( + progressTask, + log, + configuration.concurrency(), + configuration.jobId(), + taskRegistryFactory, + userLogRegistryFactory + ); + } + return progressTracker; + } + ALGO build( G graphOrGraphStore, CONFIG configuration, diff --git a/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java b/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java index 1be1d2da185..bd540bedb79 100644 --- a/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java +++ b/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java @@ -225,8 +225,8 @@ public double doubleValue(long nodeId) { } @Override - public long size() { - return property.size(); + public long nodeCount() { + return property.nodeCount(); } }; } @@ -244,8 +244,8 @@ public double doubleValue(long nodeId) { } @Override - public long size() { - return property.size(); + public long nodeCount() { + return property.nodeCount(); } }; } @@ -263,8 +263,8 @@ public double doubleValue(long nodeId) { } @Override - public long size() { - return property.size(); + public long nodeCount() { + return property.nodeCount(); } }; } diff --git a/algo-common/src/main/java/org/neo4j/gds/scaling/ScalePropertiesBaseConfig.java b/algo-common/src/main/java/org/neo4j/gds/scaling/ScalePropertiesBaseConfig.java index 7c74b650b7f..9bfafb065b8 100644 --- a/algo-common/src/main/java/org/neo4j/gds/scaling/ScalePropertiesBaseConfig.java +++ b/algo-common/src/main/java/org/neo4j/gds/scaling/ScalePropertiesBaseConfig.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.stream.Collectors; -import static org.neo4j.gds.AbstractPropertyMappings.fromObject; +import static org.neo4j.gds.PropertyMappings.fromObject; @ValueClass @SuppressWarnings("immutables:subtype") diff --git a/algo-test/src/main/java/org/neo4j/gds/test/TestAlgorithm.java b/algo-test/src/main/java/org/neo4j/gds/test/TestAlgorithm.java index 64250a7bb8c..f224286a0c1 100644 --- a/algo-test/src/main/java/org/neo4j/gds/test/TestAlgorithm.java +++ b/algo-test/src/main/java/org/neo4j/gds/test/TestAlgorithm.java @@ -41,11 +41,12 @@ public TestAlgorithm( @Override public TestAlgorithm compute() { - progressTracker.beginSubTask(); + progressTracker.beginSubTask(100); if (throwInCompute) { throw new IllegalStateException("boo"); } + progressTracker.logProgress(50); relationshipCount = graph.relationshipCount(); progressTracker.endSubTask(); diff --git a/algo/src/main/java/org/neo4j/gds/beta/indexInverse/InverseRelationships.java b/algo/src/main/java/org/neo4j/gds/beta/indexInverse/InverseRelationships.java index 3062b2f9c24..78a2bf16940 100644 --- a/algo/src/main/java/org/neo4j/gds/beta/indexInverse/InverseRelationships.java +++ b/algo/src/main/java/org/neo4j/gds/beta/indexInverse/InverseRelationships.java @@ -79,7 +79,7 @@ public Map compute() { .propertySchemasFor(fromRelationshipType); var propertyKeys = propertySchemas.stream().map(PropertySchema::key).collect(Collectors.toList()); - var relationshipsBuilder = initializeRelationshipsBuilder(propertySchemas); + var relationshipsBuilder = initializeRelationshipsBuilder(fromRelationshipType, propertySchemas); var tasks = createTasks(fromRelationshipType, propertyKeys, relationshipsBuilder); @@ -109,8 +109,9 @@ public Map compute() { } @NotNull - private RelationshipsBuilder initializeRelationshipsBuilder(List propertySchemas) { + private RelationshipsBuilder initializeRelationshipsBuilder(RelationshipType relationshipType, List propertySchemas) { RelationshipsBuilderBuilder relationshipsBuilderBuilder = GraphFactory.initRelationshipsBuilder() + .relationshipType(relationshipType) .concurrency(config.concurrency()) .nodes(graphStore.nodes()) .executorService(executorService) diff --git a/algo/src/main/java/org/neo4j/gds/beta/undirected/ToUndirected.java b/algo/src/main/java/org/neo4j/gds/beta/undirected/ToUndirected.java index 6eb1bdc7323..9d102b3d606 100644 --- a/algo/src/main/java/org/neo4j/gds/beta/undirected/ToUndirected.java +++ b/algo/src/main/java/org/neo4j/gds/beta/undirected/ToUndirected.java @@ -74,7 +74,10 @@ public SingleTypeRelationships compute() { .propertySchemasFor(fromRelationshipType); var propertyKeys = propertySchemas.stream().map(PropertySchema::key).collect(Collectors.toList()); - var relationshipsBuilder = initializeRelationshipsBuilder(propertySchemas); + var relationshipsBuilder = initializeRelationshipsBuilder( + RelationshipType.of(config.mutateRelationshipType()), + propertySchemas + ); var tasks = createTasks(fromRelationshipType, propertyKeys, relationshipsBuilder); @@ -101,8 +104,12 @@ public SingleTypeRelationships compute() { } @NotNull - private RelationshipsBuilder initializeRelationshipsBuilder(List propertySchemas) { + private RelationshipsBuilder initializeRelationshipsBuilder( + RelationshipType relationshipType, + List propertySchemas + ) { RelationshipsBuilderBuilder relationshipsBuilderBuilder = GraphFactory.initRelationshipsBuilder() + .relationshipType(relationshipType) .concurrency(config.concurrency()) .nodes(graphStore.nodes()) .executorService(executorService) diff --git a/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePath.java b/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePath.java index a2dfe2cecd1..2378f1cbd48 100644 --- a/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePath.java +++ b/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePath.java @@ -21,6 +21,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -43,12 +44,14 @@ public class CollapsePath extends Algorithm { private final List pathTemplates; private final long nodeCount; private final boolean allowSelfLoops; + private RelationshipType mutateRelationshipType; private final int concurrency; private final ExecutorService executorService; public CollapsePath( List pathTemplates, boolean allowSelfLoops, + RelationshipType mutateRelationshipType, int concurrency, ExecutorService executorService ) { @@ -56,6 +59,7 @@ public CollapsePath( this.pathTemplates = pathTemplates; this.nodeCount = pathTemplates.get(0)[0].nodeCount(); this.allowSelfLoops = allowSelfLoops; + this.mutateRelationshipType = mutateRelationshipType; this.concurrency = concurrency; this.executorService = executorService; } @@ -64,6 +68,7 @@ public CollapsePath( public SingleTypeRelationships compute() { RelationshipsBuilder relImporter = GraphFactory.initRelationshipsBuilder() .nodes(pathTemplates.get(0)[0]) // just need any arbitrary graph + .relationshipType(mutateRelationshipType) .orientation(Orientation.NATURAL) .aggregation(Aggregation.NONE) .concurrency(concurrency) diff --git a/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePathAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePathAlgorithmFactory.java index 1a69af28c0c..9321d6cd310 100644 --- a/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePathAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePathAlgorithmFactory.java @@ -64,6 +64,7 @@ public CollapsePath build( return new CollapsePath( pathTemplatesEncodedAsListsOfSingleRelationshipTypeGraphs, config.allowSelfLoops(), + RelationshipType.of(config.mutateRelationshipType()), config.concurrency(), Pools.DEFAULT ); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageHelper.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageHelper.java index 9c80628b99c..25f2f3e6d1d 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageHelper.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageHelper.java @@ -24,6 +24,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.NodeSchemaEntry; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -278,7 +279,7 @@ private static Map> propertyKeysPerNodeLabel(GraphSchema .nodeSchema() .entries() .stream() - .collect(Collectors.toMap(e -> e.identifier, e -> e.properties().keySet())); + .collect(Collectors.toMap(NodeSchemaEntry::identifier, e -> e.properties().keySet())); } private static Map> filteredPropertyKeysPerNodeLabel(Graph graph, GraphSageTrainConfig config) { diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNFactory.java b/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNFactory.java index f58ebcca91e..dacde0b0137 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNFactory.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNFactory.java @@ -118,7 +118,7 @@ public MemoryEstimation memoryEstimation(CONFIG config) { config.heterogeneous() ? dims.relationshipCounts().size() : 1 ))); - int outputDimension = config.outputDimension().orElse(FUDGED_BINARY_DIMENSION); + int outputDimension = config.outputDimension().orElse(binaryDimension); builder.perNode( "Embeddings output", n -> HugeObjectArray.memoryEstimation(n, MemoryUsage.sizeOfDoubleArray(outputDimension)) diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java index 528f89336a5..562f71b9228 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java @@ -95,15 +95,15 @@ protected boolean lessThan(long a, long b) { public LongDoubleScatterMap compute() { //Find the first node with greedy algorithm progressTracker.beginSubTask(); - greedyPart(); + var firstSeedNode = greedyPart(); //Find the next k-1 nodes using the list-sorting procedure - lazyForwardPart(); + lazyForwardPart(firstSeedNode); progressTracker.endSubTask(); return seedSetNodes; } - private void greedyPart() { + private long greedyPart() { HugeDoubleArray singleSpreadArray = HugeDoubleArray.newArray(graph.nodeCount()); progressTracker.beginSubTask(graph.nodeCount()); var tasks = PartitionUtils.rangePartition( @@ -136,15 +136,16 @@ private void greedyPart() { gain = spreads.cost(highestNode); spreads.pop(); seedSetNodes.put(highestNode, gain); + return highestNode; } - private void lazyForwardPart() { + private void lazyForwardPart(long firstSeedNode) { var independentCascade = ICLazyForwardMC.create( graph, propagationProbability, monteCarloSimulations, - seedSetNodes.keys[0], + firstSeedNode, (int) seedSetCount, concurrency, executorService, diff --git a/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java b/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java index fbd2e4450ae..19a08ab1ad9 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java @@ -53,22 +53,22 @@ class GraphAggregationPhase { static MemoryEstimation memoryEstimation() { return MemoryEstimations.builder(GraphAggregationPhase.class) - .rangePerGraphDimension("aggregated graph", (rootGraphDimensions, concurrency) -> { + .rangePerGraphDimension("aggregated graph", (rootDimensions, concurrency) -> { // The input graph might have multiple node and relationship properties // but the aggregated graph will never have more than a single relationship property // so let's - var maxDimensions = ImmutableGraphDimensions - .builder() - .from(rootGraphDimensions) - .build(); + + // Handle the case where the input graph has only one node + var minNodeCount = Math.min(2, rootDimensions.nodeCount()); + var minRelCount = Math.min(1, rootDimensions.relCountUpperBound()); var minDimensions = ImmutableGraphDimensions .builder() - .nodeCount(2) - .highestPossibleNodeCount(2) - .relationshipCounts(Map.of(RelationshipType.of("foo"), 1L)) - .relCountUpperBound(1) - .highestRelationshipId(1) + .nodeCount(minNodeCount) + .highestPossibleNodeCount(minNodeCount) + .relationshipCounts(Map.of(RelationshipType.of("foo"), minRelCount)) + .relCountUpperBound(minRelCount) + .highestRelationshipId(minRelCount) .build(); var relationshipProjections = ImmutableRelationshipProjections.builder() @@ -89,7 +89,7 @@ static MemoryEstimation memoryEstimation() { false ); var min = memoryEstimation.estimate(minDimensions, concurrency).memoryUsage().min; - var max = memoryEstimation.estimate(maxDimensions, concurrency).memoryUsage().max; + var max = memoryEstimation.estimate(rootDimensions, concurrency).memoryUsage().max; return MemoryRange.of(min, max); }).perNode("sorted communities", HugeLongArray::memoryEstimation) @@ -148,6 +148,7 @@ Graph run() { IdMap idMap = nodesBuilder.build().idMap(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(RelationshipType.of("_IGNORED_")) .orientation(direction.toOrientation()) .addPropertyConfig(GraphFactory.PropertyConfig.builder() .propertyKey("property") diff --git a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java index 0244ea21170..784ce0cbf41 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java @@ -133,9 +133,10 @@ public LeidenResult compute() { gamma, concurrency ); - var communitiesCount = localMovePhase.run(); - boolean localPhaseConverged = communitiesCount == workingGraph.nodeCount() || localMovePhase.swaps == 0; + localMovePhase.run(); + //if you do swaps, no convergence + boolean localPhaseConverged = localMovePhase.swaps == 0; progressTracker.endSubTask("Local Move"); progressTracker.beginSubTask("Modularity Computation"); @@ -244,11 +245,11 @@ public LeidenResult compute() { @NotNull private LeidenResult getLeidenResult(boolean didConverge, int iteration) { - boolean seedIsOptimal = didConverge && seedValues.isPresent() && iteration == 0; - if (seedIsOptimal) { + boolean stoppedAtFirstIteration = didConverge && iteration == 0; + if (stoppedAtFirstIteration) { var modularity = modularities[0]; return LeidenResult.of( - LeidenUtils.createSeedCommunities(rootGraph.nodeCount(), seedValues.orElse(null)), + LeidenUtils.createStartingCommunities(rootGraph.nodeCount(), seedValues.orElse(null)), 1, didConverge, null, @@ -266,8 +267,8 @@ private LeidenResult getLeidenResult(boolean didConverge, int iteration) { ); } } - - private boolean updateModularity( + + private void updateModularity( Graph workingGraph, HugeLongArray localMoveCommunities, HugeDoubleArray localMoveCommunityVolumes, @@ -276,8 +277,10 @@ private boolean updateModularity( boolean localPhaseConverged, int iteration ) { - boolean seedIsOptimal = localPhaseConverged && seedValues.isPresent() && iteration == 0; - boolean shouldCalculateModularity = !localPhaseConverged || seedIsOptimal; + // Will calculate modularity only if: + // - the local phase has not converged (i.e., no swaps done) + // - or we terminate in the first iteration (i.e., given seeding is optimal, graph is empty, etc) + boolean shouldCalculateModularity = !localPhaseConverged || iteration == 0; if (shouldCalculateModularity) { modularities[iteration] = ModularityComputer.compute( @@ -291,7 +294,6 @@ private boolean updateModularity( progressTracker ); } - return seedIsOptimal; } private double initVolumes( @@ -327,7 +329,7 @@ private double initVolumes( rootGraph.forEachNode(nodeId -> { long communityId = initialCommunities.get(nodeId); progressTracker.logProgress(); - communityVolumes.addTo(communityId, rootGraph.degree(nodeId)); + communityVolumes.addTo(communityId, nodeVolumes.get(nodeId)); return true; }); progressTracker.endSubTask("Initialization"); diff --git a/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java b/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java index ad2d577bbb8..9f7a25afe5b 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.leiden; -import org.apache.commons.lang3.mutable.MutableDouble; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.mem.MemoryEstimation; @@ -72,9 +71,7 @@ static LocalMovePhase create( double gamma, int concurrency ) { - var encounteredCommunitiesWeights = HugeDoubleArray.newArray(graph.nodeCount()); - encounteredCommunitiesWeights.setAll(c -> -1L); - + return new LocalMovePhase( graph, seedCommunities, @@ -103,10 +100,9 @@ private LocalMovePhase( } /** - * * @return The new community count. */ - public long run() { + public void run() { HugeAtomicDoubleArray atomicCommunityVolumes = HugeAtomicDoubleArray.newArray(graph.nodeCount()); graph.forEachNode(v -> { atomicCommunityVolumes.set(v, communityVolumes.get(v)); @@ -147,15 +143,11 @@ public long run() { swaps += task.swaps; } - MutableDouble aliveCommunities = new MutableDouble(graph.nodeCount()); graph.forEachNode(v -> { communityVolumes.set(v, atomicCommunityVolumes.get(v)); - if (Double.compare(communityVolumes.get(v), 0.0) == 0) { - aliveCommunities.decrement(); - } return true; }); - return aliveCommunities.longValue(); + } - + } diff --git a/algo/src/main/java/org/neo4j/gds/leiden/LocalMoveTask.java b/algo/src/main/java/org/neo4j/gds/leiden/LocalMoveTask.java index 0b6f42624cd..58f37752ab0 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/LocalMoveTask.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/LocalMoveTask.java @@ -129,8 +129,11 @@ private void moveNodeToNewCommunity( long oldCommunityId = currentCommunities.get(nodeId); currentCommunities.set(nodeId, newCommunityId); communityVolumes.getAndAdd(newCommunityId, currentNodeVolume); - communityVolumes.getAndAdd(oldCommunityId, -currentNodeVolume); - + //do a atomic update to never go into negatives etc + communityVolumes.update(oldCommunityId, (oldValue) -> { + var diff = oldValue - currentNodeVolume; + return Math.max(diff, 0.0); + }); swaps++; } diff --git a/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java b/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java index 62b15fcad94..2754623084e 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java @@ -20,6 +20,7 @@ package org.neo4j.gds.louvain; import org.neo4j.gds.Algorithm; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.RelationshipIterator; @@ -229,13 +230,11 @@ private Graph summarizeGraph( }); terminationFlag.assertRunning(); - double scaleCoefficient = 1.0; - if (workingGraph.schema().isUndirected()) { - scaleCoefficient /= 2.0; - } + IdMap idMap = nodesBuilder.build().idMap(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(RelationshipType.of("IGNORED")) .orientation(rootGraph.schema().direction().toOrientation()) .addPropertyConfig(GraphFactory.PropertyConfig.builder() .propertyKey("property") @@ -244,7 +243,6 @@ private Graph summarizeGraph( .executorService(executorService) .build(); - double finalScaleCoefficient = scaleCoefficient; var relationshipCreators = PartitionUtils.rangePartition( concurrency, workingGraph.nodeCount(), @@ -253,7 +251,6 @@ private Graph summarizeGraph( relationshipsBuilder, modularityOptimization, workingGraph.concurrentCopy(), - finalScaleCoefficient, partition ), Optional.empty() @@ -315,21 +312,16 @@ static final class RelationshipCreator implements Runnable { private final Partition partition; - private final double scaleCoefficient; - - private RelationshipCreator( RelationshipsBuilder relationshipsBuilder, ModularityOptimization modularityOptimization, RelationshipIterator relationshipIterator, - double scaleCoefficient, Partition partition ) { this.relationshipsBuilder = relationshipsBuilder; this.modularityOptimization = modularityOptimization; this.relationshipIterator = relationshipIterator; this.partition = partition; - this.scaleCoefficient = scaleCoefficient; } @Override @@ -337,18 +329,13 @@ public void run() { partition.consume(nodeId -> { long communityId = modularityOptimization.getCommunityId(nodeId); relationshipIterator.forEachRelationship(nodeId, 1.0, (source, target, property) -> { - // In the case of undirected graphs, we need scaling as otherwise we'd have double the value in these edges - // see GraphAggregationPhase.java for the equivalent in Leiden; and the corresponding test - if (source == target) { - relationshipsBuilder.add(communityId, modularityOptimization.getCommunityId(target), property); - } else { + //ignore scaling alltogether relationshipsBuilder.add( communityId, modularityOptimization.getCommunityId(target), - property * scaleCoefficient + property ); - } return true; }); diff --git a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java index b9e6ed11e2f..5daf9e58aae 100644 --- a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java +++ b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java @@ -19,13 +19,13 @@ */ package org.neo4j.gds.modularity; +import com.carrotsearch.hppc.cursors.LongLongCursor; import org.apache.commons.lang3.mutable.MutableDouble; -import org.apache.commons.lang3.mutable.MutableLong; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.concurrency.RunWithConcurrency; -import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeAtomicDoubleArray; +import org.neo4j.gds.core.utils.paged.HugeLongLongMap; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.partition.PartitionUtils; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -38,17 +38,32 @@ public class ModularityCalculator extends Algorithm { private final Graph graph; private final LongUnaryOperator communityIdProvider; + private final HugeLongLongMap communityMapper; private final int concurrency; + public static ModularityCalculator create( + Graph graph, + LongUnaryOperator seedCommunityIdProvider, + int concurrency + ) { + var communityMapper = createMapping(graph.nodeCount(), seedCommunityIdProvider); + LongUnaryOperator communityIdProvider = nodeId -> communityMapper.getOrDefault( + seedCommunityIdProvider.applyAsLong(nodeId), + -1 + ); + return new ModularityCalculator(graph, communityIdProvider, communityMapper, concurrency); + } - protected ModularityCalculator( + private ModularityCalculator( Graph graph, LongUnaryOperator communityIdProvider, + HugeLongLongMap communityMapper, int concurrency ) { super(ProgressTracker.NULL_TRACKER); this.graph = graph; this.communityIdProvider = communityIdProvider; + this.communityMapper = communityMapper; this.concurrency = concurrency; } @@ -56,9 +71,9 @@ protected ModularityCalculator( public ModularityResult compute() { var nodeCount = graph.nodeCount(); - var insideRelationships = HugeAtomicDoubleArray.newArray(nodeCount); - var totalCommunityRelationships = HugeAtomicDoubleArray.newArray(nodeCount); - var communityTracker = HugeAtomicBitSet.create(nodeCount); + var communityCount = communityMapper.size(); + var insideRelationships = HugeAtomicDoubleArray.newArray(communityCount); + var totalCommunityRelationships = HugeAtomicDoubleArray.newArray(communityCount); var totalRelationshipWeight = new DoubleAdder(); var tasks = PartitionUtils.rangePartition( @@ -69,7 +84,6 @@ public ModularityResult compute() { graph, insideRelationships, totalCommunityRelationships, - communityTracker, communityIdProvider, totalRelationshipWeight ), Optional.empty() @@ -80,25 +94,38 @@ public ModularityResult compute() { .tasks(tasks) .run(); - var communityCount = communityTracker.cardinality(); var communityModularities = HugeObjectArray.newArray( CommunityModularity.class, communityCount ); var totalRelWeight = totalRelationshipWeight.doubleValue(); - var resultTracker = new MutableLong(); var totalModularity = new MutableDouble(); - communityTracker.forEachSetBit(communityId -> { - var ec = insideRelationships.get(communityId); - var Kc = totalCommunityRelationships.get(communityId); + long resultIndex = 0; + for (LongLongCursor cursor : communityMapper) { + long communityId = cursor.key; + long mappedCommunityId = cursor.value; + var ec = insideRelationships.get(mappedCommunityId); + var Kc = totalCommunityRelationships.get(mappedCommunityId); var modularity = (ec - Kc * Kc * (1.0 / totalRelWeight)) / totalRelWeight; totalModularity.add(modularity); - communityModularities.set(resultTracker.getAndIncrement(), CommunityModularity.of(communityId, modularity)); - }); + communityModularities.set(resultIndex++, CommunityModularity.of(communityId, modularity)); + } return ModularityResult.of(totalModularity.doubleValue(), communityCount, communityModularities); } @Override public void release() {} + static HugeLongLongMap createMapping(long nodeCount, LongUnaryOperator seedCommunityId) { + + var seedMap = new HugeLongLongMap(nodeCount); + long seedId = 0; + for (long nodeId = 0; nodeId < nodeCount; ++nodeId) { + long communityId = seedCommunityId.applyAsLong(nodeId); + if (!seedMap.containsKey(communityId)) { + seedMap.put(communityId, seedId++); + } + } + return seedMap; + } } diff --git a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculatorFactory.java b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculatorFactory.java index 7797c171606..494c9cf3ac2 100644 --- a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculatorFactory.java +++ b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculatorFactory.java @@ -30,7 +30,7 @@ public ModularityCalculator build( CONFIG configuration, ProgressTracker progressTracker ) { - return new ModularityCalculator( + return ModularityCalculator.create( graph, graph.nodeProperties(configuration.communityProperty())::longValue, configuration.concurrency() diff --git a/algo/src/main/java/org/neo4j/gds/modularity/RelationshipCountCollector.java b/algo/src/main/java/org/neo4j/gds/modularity/RelationshipCountCollector.java index 016d2d078d1..2483f5ba9fe 100644 --- a/algo/src/main/java/org/neo4j/gds/modularity/RelationshipCountCollector.java +++ b/algo/src/main/java/org/neo4j/gds/modularity/RelationshipCountCollector.java @@ -20,7 +20,6 @@ package org.neo4j.gds.modularity; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeAtomicDoubleArray; import org.neo4j.gds.core.utils.partition.Partition; @@ -32,7 +31,6 @@ class RelationshipCountCollector implements Runnable { private final Graph localGraph; private final HugeAtomicDoubleArray insideRelationships; private final HugeAtomicDoubleArray totalCommunityRelationships; - private final HugeAtomicBitSet communityTracker; private final LongUnaryOperator communityIdProvider; private final DoubleAdder totalRelationshipWeight; @@ -42,19 +40,15 @@ class RelationshipCountCollector implements Runnable { Graph graph, HugeAtomicDoubleArray insideRelationships, HugeAtomicDoubleArray totalCommunityRelationships, - HugeAtomicBitSet communityTracker, LongUnaryOperator communityIdProvider, DoubleAdder totalRelationshipWeight ) { this.totalRelationshipWeight = totalRelationshipWeight; - assert insideRelationships.size() == totalCommunityRelationships.size() - && insideRelationships.size() == communityTracker.size(); - + assert insideRelationships.size() == totalCommunityRelationships.size(); this.partition = partition; this.localGraph = graph.concurrentCopy(); this.insideRelationships = insideRelationships; this.totalCommunityRelationships = totalCommunityRelationships; - this.communityTracker = communityTracker; this.communityIdProvider = communityIdProvider; } @@ -66,7 +60,6 @@ public void run() { long communityId = communityIdProvider.applyAsLong(nodeId); localGraph.forEachRelationship(nodeId, 1.0, (s, t, w) -> { long tCommunityId = communityIdProvider.applyAsLong(t); - communityTracker.set(communityId); if (tCommunityId == communityId) { insideRelationships.getAndAdd(communityId, w); } diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java index 686206e2346..546b16818ee 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java @@ -414,10 +414,9 @@ public LongNodePropertyValues asNodeProperties() { public long longValue(long nodeId) { return getCommunityId(nodeId); } - - @Override - public long size() { - return currentCommunities.size(); + @Override + public long nodeCount() { + return graph.nodeCount(); } }; } diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java b/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java index 60cfced65c3..be38380605f 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java @@ -115,7 +115,7 @@ public boolean masterCompute(MasterComputeContext context) { var properties = new DoubleNodePropertyValues() { @Override - public long size() { + public long nodeCount() { return context.nodeCount(); } diff --git a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaStepping.java b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaStepping.java index 23a38e6feac..7bf85bac8df 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaStepping.java +++ b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaStepping.java @@ -58,6 +58,8 @@ public final class DeltaStepping extends Algorithm { private static final int NO_BIN = Integer.MAX_VALUE; private static final int BIN_SIZE_THRESHOLD = 1000; + private static final int BATCH_SIZE = 64; + private final Graph graph; private final long startNode; @@ -281,8 +283,8 @@ int minNonEmptyBin() { private void relaxGlobalBin() { long offset; - while ((offset = frontierIndex.getAndAdd(64)) < frontierLength) { - long limit = Math.min(offset + 64, frontierLength); + while ((offset = frontierIndex.getAndAdd(BATCH_SIZE)) < frontierLength) { + long limit = Math.min(offset + BATCH_SIZE, frontierLength); for (long idx = offset; idx < limit; idx++) { var nodeId = frontier.get(idx); @@ -326,7 +328,8 @@ private void relaxNode(long nodeId) { break; } // CAX failed, retry - oldDist = witness; + //we need to fetch the most recent value from distances + oldDist = distances.distance(targetNodeId); } return true; diff --git a/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java b/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java index de82b067f5c..3870a66d40c 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java +++ b/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java @@ -168,24 +168,38 @@ public double compareAndExchange(long nodeId, double expectedDistance, double ne // locked by another thread if (currentPredecessor < 0) { - return distances.get(nodeId); + //we should signal failure + // for that we must be sure not to return the 'expectedDistance' by accident! + //we hence return its negation (or -1 if ==0) + return (Double.compare(expectedDistance, 0.0) == 0) ? -1.0 : -expectedDistance; } var witness = predecessors.compareAndExchange(nodeId, currentPredecessor, -predecessor - 1); // CAX failed if (witness != currentPredecessor) { - return distances.get(nodeId); + //we should signal failure + // for that we must be sure not to return the 'expectedDistance' by accident! + return (Double.compare(expectedDistance, 0.0) == 0) ? -1.0 : -expectedDistance; } - // we have the look - distances.set(nodeId, newDistance); + // we have the lock; no-one else can write on nodeId at the moment. + // Let us do a check if it makes sense to update - // unlock - predecessors.set(nodeId, predecessor); + double oldDistance = distances.get(nodeId); + + if (oldDistance > newDistance) { + distances.set(nodeId, newDistance); + // unlock + predecessors.set(nodeId, predecessor); + // return previous distance to signal successful CAX - // return previous distance to signal successful CAX - return expectedDistance; + return expectedDistance; + } + predecessors.set(nodeId, currentPredecessor); + //signal unsuccesful update + //note that this unsuccesful update will be the last attempt + return (Double.compare(expectedDistance, 0.0) == 0.0) ? -1.0 : -expectedDistance; } } } diff --git a/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java b/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java index 203721de43d..223d9c166e8 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java +++ b/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java @@ -217,6 +217,7 @@ private PathResult next(TraversalPredicate traversalPredicate, ImmutablePathResu return pathResult(node, pathResultBuilder); } } + return PathResult.EMPTY; } diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java b/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java index 27efd1e401c..8d5e43eb081 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java @@ -31,6 +31,7 @@ */ final class MutablePathResult { + private final long[] EMPTY_ARRAY = new long[0]; private long index; private final long sourceNode; @@ -131,10 +132,9 @@ boolean matches(MutablePathResult path, int index) { */ boolean matchesExactly(MutablePathResult path, int index) { - if (relationshipIds == null || path.relationshipIds == null) { + if (relationshipIds.length == 0 || path.relationshipIds.length == 0) { return matches(path, index); } - for (int i = 0; i < index; i++) { if (nodeIds[i] != path.nodeIds[i]) { return false; @@ -157,7 +157,8 @@ boolean matchesExactly(MutablePathResult path, int index) { * The cost value associated with the last value in this path, is added to * the costs for each node in the second path. */ - void append(MutablePathResult path) { + + private void append(MutablePathResult path, long[] relationships) { // spur node is end of first and beginning of second path assert nodeIds[nodeIds.length - 1] == path.nodeIds[0]; @@ -166,15 +167,10 @@ void append(MutablePathResult path) { var newNodeIds = new long[oldLength + path.nodeIds.length - 1]; var newCosts = new double[oldLength + path.nodeIds.length - 1]; - var oldRelationshipIdsLength = relationshipIds.length; - var newRelationshipIds = new long[oldRelationshipIdsLength + path.relationshipIds.length]; - // copy node ids System.arraycopy(this.nodeIds, 0, newNodeIds, 0, oldLength); System.arraycopy(path.nodeIds, 1, newNodeIds, oldLength, path.nodeIds.length - 1); - // copy relationship ids - System.arraycopy(this.relationshipIds, 0, newRelationshipIds, 0, oldRelationshipIdsLength); - System.arraycopy(path.relationshipIds, 0, newRelationshipIds, oldRelationshipIdsLength, path.relationshipIds.length); + // copy costs System.arraycopy(this.costs, 0, newCosts, 0, oldLength); System.arraycopy(path.costs, 1, newCosts, oldLength, path.costs.length - 1); @@ -186,10 +182,41 @@ void append(MutablePathResult path) { } this.nodeIds = newNodeIds; - this.relationshipIds = newRelationshipIds; + this.relationshipIds = relationships; this.costs = newCosts; } + void append(MutablePathResult path) { + + var oldRelationshipIdsLength = relationshipIds.length; + var newRelationshipIds = new long[oldRelationshipIdsLength + path.relationshipIds.length]; + // copy relationship ids + System.arraycopy(this.relationshipIds, 0, newRelationshipIds, 0, oldRelationshipIdsLength); + System.arraycopy( + path.relationshipIds, + 0, + newRelationshipIds, + oldRelationshipIdsLength, + path.relationshipIds.length + ); + + append(path, newRelationshipIds); + } + + /** + * Appends the given path to this path without creating an explicit relationship array. + * + * The last node in this path, must match the first node in the given path. + * This node will only appear once in the resulting path. + * The cost value associated with the last value in this path, is added to + * the costs for each node in the second path. + */ + void appendWithoutRelationshipIds(MutablePathResult path) { + // spur node is end of first and beginning of second path + append(path, EMPTY_ARRAY); + } + + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index 2bd70bbb9a6..8ecc2700b17 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -39,6 +39,8 @@ import java.util.Comparator; import java.util.Optional; import java.util.PriorityQueue; +import java.util.function.BiConsumer; +import java.util.function.ToLongBiFunction; import java.util.stream.Stream; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -50,9 +52,12 @@ public final class Yens extends Algorithm { private final Graph graph; private final ShortestPathYensBaseConfig config; private final Dijkstra dijkstra; + private final LongScatterSet nodeAvoidList; + private final LongObjectScatterMap relationshipAvoidList; + private final ToLongBiFunction + relationshipAvoidMapper; + private final BiConsumer pathAppender; - private final LongScatterSet nodeBlackList; - private final LongObjectScatterMap relationshipBlackList; /** * Configure Yens to compute at most one source-target shortest path. @@ -63,12 +68,15 @@ public static Yens sourceTarget( ProgressTracker progressTracker ) { // If the input graph is a multi-graph, we need to track - // parallel relationships. This is necessary since shortest + // parallel relationships ids. This is necessary since shortest // paths can visit the same nodes via different relationships. + //If not, we need to track which is the next neighbor. + + boolean shouldTrackRelationships = graph.isMultiGraph(); var newConfig = ImmutableShortestPathYensBaseConfig .builder() .from(config) - .trackRelationships(graph.isMultiGraph()) + .trackRelationships(shouldTrackRelationships) .build(); // Init dijkstra algorithm for computing shortest paths var dijkstra = Dijkstra.sourceTarget(graph, newConfig, Optional.empty(), progressTracker); @@ -95,16 +103,37 @@ private Yens(Graph graph, Dijkstra dijkstra, ShortestPathYensBaseConfig config, this.config = config; // Track nodes and relationships that are skipped in a single iteration. // The content of these data structures is reset after each of k iterations. - this.nodeBlackList = new LongScatterSet(); - this.relationshipBlackList = new LongObjectScatterMap<>(); - // set filter in Dijkstra to respect our blacklists + this.nodeAvoidList = new LongScatterSet(); + this.relationshipAvoidList = new LongObjectScatterMap<>(); + // set filter in Dijkstra to respect our list of relationships to avoid this.dijkstra = dijkstra; + + if (config.trackRelationships()) { + // if we are in a multi-graph, we must store the relationships ids as they are + //since two nodes may be connected by multiple relationships and we must know which to avoid + relationshipAvoidMapper = (path, position) -> path.relationship(position); + pathAppender = (rootPath, spurPath) -> rootPath.append(MutablePathResult.of(spurPath)); + } else { + //otherwise the graph has surely no parallel edges, we do not need to explicitly store relationship ids + //we can just store endpoints, so that we know which nodes a node should avoid + relationshipAvoidMapper = (path, position) -> path.node(position + 1); + pathAppender = (rootPath, spurPath) -> rootPath.appendWithoutRelationshipIds(MutablePathResult.of(spurPath)); + } dijkstra.withRelationshipFilter((source, target, relationshipId) -> - !nodeBlackList.contains(target) && - !(relationshipBlackList.getOrDefault(source, EMPTY_SET).contains(relationshipId)) + !nodeAvoidList.contains(target) + && !shouldAvoidRelationship(source, target, relationshipId) + ); } + private boolean shouldAvoidRelationship(long source, long target, long relationshipId) { + long forbidden = config.trackRelationships() + ? relationshipId + : target; + return relationshipAvoidList.getOrDefault(source, EMPTY_SET).contains(forbidden); + + } + @Override public DijkstraResult compute() { progressTracker.beginSubTask(); @@ -139,13 +168,13 @@ public DijkstraResult compute() { // Filter relationships that are part of the previous // shortest paths which share the same root path. if (rootPath.matchesExactly(path, n + 1)) { - var relationshipId = path.relationship(n); + var relationshipId = relationshipAvoidMapper.applyAsLong(path, n); - var neighbors = relationshipBlackList.get(spurNode); + var neighbors = relationshipAvoidList.get(spurNode); if (neighbors == null) { neighbors = new LongHashSet(); - relationshipBlackList.put(spurNode, neighbors); + relationshipAvoidList.put(spurNode, neighbors); } neighbors.add(relationshipId); } @@ -153,7 +182,7 @@ public DijkstraResult compute() { // Filter nodes from root path to avoid cyclic path searches. for (int j = 0; j < n; j++) { - nodeBlackList.add(rootPath.node(j)); + nodeAvoidList.add(rootPath.node(j)); } // Calculate the spur path from the spur node to the sink. @@ -162,8 +191,8 @@ public DijkstraResult compute() { var spurPath = computeDijkstra(graph.toOriginalNodeId(spurNode)); // Clear filters for next spur node - nodeBlackList.clear(); - relationshipBlackList.clear(); + nodeAvoidList.clear(); + relationshipAvoidList.clear(); // No new candidate from this spur node, continue with next node. if (spurPath.isEmpty()) { @@ -171,7 +200,8 @@ public DijkstraResult compute() { } // Entire path is made up of the root path and spur path. - rootPath.append(MutablePathResult.of(spurPath.get())); + pathAppender.accept(rootPath, spurPath.get()); + // Add the potential k-shortest path to the heap. if (!candidates.contains(rootPath)) { candidates.add(rootPath); @@ -203,8 +233,8 @@ private PriorityQueue initCandidatesQueue() { @Override public void release() { dijkstra.release(); - nodeBlackList.release(); - relationshipBlackList.release(); + nodeAvoidList.release(); + relationshipAvoidList.release(); } private Optional computeDijkstra(long sourceNode) { diff --git a/algo/src/main/java/org/neo4j/gds/similarity/SimilarityGraphBuilder.java b/algo/src/main/java/org/neo4j/gds/similarity/SimilarityGraphBuilder.java index 9eddd7ab32c..241e0468d5b 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/SimilarityGraphBuilder.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/SimilarityGraphBuilder.java @@ -20,6 +20,7 @@ package org.neo4j.gds.similarity; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.compress.AdjacencyListBehavior; @@ -84,6 +85,7 @@ public SimilarityGraphBuilder( public Graph build(Stream stream) { var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap.rootIdMap()) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of("property")) .concurrency(concurrency) diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java index c4a5441d076..9e4940c9143 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java @@ -106,7 +106,11 @@ public MemoryEstimation memoryEstimation(CONFIG config) { ); } if (config.hasTopN()) { - builder.add("topN list", TopNList.memoryEstimation(topN)); + builder.add( + "topN list", + MemoryEstimations.setup("", (dimensions, concurrency) -> + TopNList.memoryEstimation(dimensions.nodeCount(), topN)) + ); } return builder.build(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java index 4f79cba6924..5295efeab76 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java @@ -55,16 +55,22 @@ static final class SortedLongArrayPropertyValues implements LongArrayNodePropert private final HugeObjectArray properties; SortedLongArrayPropertyValues(NodePropertyValues nodePropertyValues) { - this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.size()); + this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.nodeCount()); this.properties.setAll(i -> { - var value = nodePropertyValues.longArrayValue(i).clone(); + long[] input = nodePropertyValues.longArrayValue(i); + + if (input == null) { + return null; + } + + var value = input.clone(); Arrays.parallelSort(value); return value; }); } @Override - public long size() { + public long nodeCount() { return properties.size(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java index 2142b87109e..c1e37ddd4e7 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java @@ -86,8 +86,8 @@ public Value value(long nodeId) { } @Override - public long size() { - return properties.size(); + public long nodeCount() { + return properties.nodeCount(); } private void check(long nodeId, @Nullable Object value) { diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java index dad85ebf44a..00d7bcf40a6 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java @@ -33,7 +33,6 @@ import org.neo4j.gds.similarity.SimilarityResult; import org.neo4j.gds.similarity.filtering.NodeFilter; -import java.util.Arrays; import java.util.Comparator; import java.util.Objects; import java.util.Optional; @@ -57,7 +56,6 @@ public class NodeSimilarity extends Algorithm { private final MetricSimilarityComputer similarityComputer; private HugeObjectArray vectors; private HugeObjectArray weights; - private long nodesToCompare; private final boolean weighted; @@ -188,7 +186,7 @@ public SimilarityGraphResult computeToGraph() { executorService ).build(similarities); } - return new SimilarityGraphResult(similarityGraph, nodesToCompare, isTopKGraph); + return new SimilarityGraphResult(similarityGraph, sourceNodes.cardinality(), isTopKGraph); } private void prepare() { @@ -218,19 +216,19 @@ private void prepare() { // TODO: we don't need to do the rest of the prepare for a node that isn't going to be used in the computation progressTracker.logProgress(graph.degree(node)); vectorComputer.forEachRelationship(node); + + if (sortVectors) { + vectorComputer.sortTargetIds(); + } if (weighted) { weights.set(node, vectorComputer.getWeights()); } - if (sortVectors) { - Arrays.sort(vectorComputer.targetIds.buffer); - } return vectorComputer.targetIds.buffer; } progressTracker.logProgress(graph.degree(node)); return null; }); - nodesToCompare = sourceNodes.cardinality(); progressTracker.endSubTask(); } @@ -437,8 +435,14 @@ private LongStream checkProgress(LongStream stream) { } private long calculateWorkload() { - long workload = nodesToCompare * nodesToCompare; - if (concurrency == 1) { + //for each source node, examine all their target nodes + //if no filter then sourceNodes == targetNodes + long workload = sourceNodes.cardinality() * targetNodes.cardinality(); + + //when on concurrency of 1 on not-filtered similarity, we only compare nodeId with greater indexed nodes + // so work is halved. This does not hold for filtered similarity, since the targetNodes might be lesser indexed. + boolean isNotFiltered = sourceNodeFilter.equals(NodeFilter.noOp) && targetNodeFilter.equals(NodeFilter.noOp); + if (concurrency == 1 && isNotFiltered) { workload = workload / 2; } return workload; diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java index 2f7e5c273d0..4da718731e9 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java @@ -101,7 +101,11 @@ public MemoryEstimation memoryEstimation(CONFIG config) { ); } if (config.hasTopN()) { - builder.add("topN list", TopNList.memoryEstimation(topN)); + builder.add( + "topN list", + MemoryEstimations.setup("", (dimensions, concurrency) -> + TopNList.memoryEstimation(dimensions.nodeCount(), topN)) + ); } return builder.build(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopKMap.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopKMap.java index 4bfde4aa305..335d66e044e 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopKMap.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopKMap.java @@ -39,10 +39,11 @@ public class TopKMap { private final BitSet sourceNodes; public static MemoryEstimation memoryEstimation(long nodes, int topK) { + int actualTopK = Math.toIntExact(Math.min(topK, nodes)); return MemoryEstimations.builder(TopKMap.class) .add("topK lists", MemoryEstimations.builder("topK lists", TopKList.class) - .add("queues", BoundedLongPriorityQueue.memoryEstimation(topK)) + .add("queues", BoundedLongPriorityQueue.memoryEstimation(actualTopK)) .build() .times(nodes) ) diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopNList.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopNList.java index ed3f7810d4e..f1cff556e7b 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopNList.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopNList.java @@ -31,9 +31,14 @@ public class TopNList { - public static MemoryEstimation memoryEstimation(int topN) { + public static MemoryEstimation memoryEstimation(long nodeCount, int topN) { + int actualTopN = topN; + if (topN > nodeCount) { //avoid overflows for nodeCount * nodeCount (when nodeCount is >2^31) + long normalizedMaximum = nodeCount * nodeCount; + actualTopN = Math.toIntExact(Math.min(normalizedMaximum, topN)); + } return MemoryEstimations.builder(TopNList.class) - .add("queue", BoundedLongLongPriorityQueue.memoryEstimation(topN)) + .add("queue", BoundedLongLongPriorityQueue.memoryEstimation(actualTopN)) .build(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/VectorComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/VectorComputer.java index bd48940112f..8c75e130e6c 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/VectorComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/VectorComputer.java @@ -25,9 +25,11 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.RelationshipConsumer; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; +import org.neo4j.gds.core.utils.TwoArraysSort; -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; +import java.util.Arrays; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; abstract class VectorComputer { final Graph graph; @@ -69,6 +71,8 @@ static VectorComputer of( : new UnweightedVectorComputer(graph); } + abstract void sortTargetIds(); + static final class UnweightedVectorComputer extends VectorComputer implements RelationshipConsumer { UnweightedVectorComputer(Graph graph) { @@ -91,6 +95,11 @@ public double[] getWeights() { )); } + @Override + void sortTargetIds() { + Arrays.sort(targetIds.buffer); + } + @Override void forEachRelationship(long node) { graph.forEachRelationship(node, this); @@ -129,5 +138,11 @@ void reset(int degree) { super.reset(degree); weights = new DoubleArrayList(degree, ARRAY_SIZING_STRATEGY); } + + @Override + void sortTargetIds() { + int length = weights.buffer.length; + TwoArraysSort.sortDoubleArrayByLongValues(targetIds.buffer, weights.buffer, length); + } } } diff --git a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java index 47781805f74..3a1716cc8aa 100644 --- a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java +++ b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java @@ -20,7 +20,7 @@ package org.neo4j.gds.spanningtree; import org.apache.commons.lang3.builder.EqualsBuilder; -import org.neo4j.gds.api.RelationshipConsumer; +import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -62,7 +62,9 @@ public double totalWeight() { return totalWeight; } - public HugeLongArray parentArray() {return parent;} + public HugeLongArray parentArray() { + return parent; + } public long parent(long nodeId) {return parent.get(nodeId);} @@ -70,13 +72,14 @@ public double costToParent(long nodeId) { return costToParent.get(nodeId); } - public void forEach(RelationshipConsumer consumer) { + public void forEach(RelationshipWithPropertyConsumer consumer) { for (int i = 0; i < nodeCount; i++) { - final long parent = this.parent.get(i); + long parent = this.parent.get(i); + double cost = this.costToParent(i); if (parent == -1) { continue; } - if (!consumer.accept(parent, i)) { + if (!consumer.accept(parent, i, cost)) { return; } } diff --git a/algo/src/main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java b/algo/src/main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java index eebbe85cd17..de059b4a2a0 100644 --- a/algo/src/main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java +++ b/algo/src/main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java @@ -150,6 +150,7 @@ private void relaxNode(long nodeId) { private void tryToUpdate(long sourceNodeId, long targetNodeId, double weight) { var oldDist = distances.distance(targetNodeId); var newDist = distances.distance(sourceNodeId) + weight; + while (Double.compare(newDist, oldDist) < 0) { var witness = distances.compareAndExchange(targetNodeId, oldDist, newDist, sourceNodeId); @@ -167,7 +168,7 @@ private void tryToUpdate(long sourceNodeId, long targetNodeId, double weight) { break; } // CAX failed, retry - oldDist = witness; + oldDist = distances.distance(targetNodeId); } smallestConsideredDistance = Math.min(newDist, smallestConsideredDistance); diff --git a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java index 3f07c4b0993..179210d398a 100644 --- a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java +++ b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java @@ -38,6 +38,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -96,7 +97,7 @@ public Stream compute() { ? new NextNodeSupplier.GraphNodeSupplier(graph.nodeCount()) : NextNodeSupplier.ListNodeSupplier.of(config, graph); - var terminationFlag = new ExternalTerminationFlag(this.terminationFlag); + var terminationFlag = new ExternalTerminationFlag(this); BlockingQueue walks = new ArrayBlockingQueue<>(config.walkBufferSize()); long[] TOMB = new long[0]; @@ -178,7 +179,10 @@ private void tasksRunner( progressTracker.endSubTask("create walks"); try { - walks.put(tombstone); + boolean finished = false; + while (!finished && terminationFlag.running()) { + finished = walks.offer(tombstone, 100, TimeUnit.MILLISECONDS); + } } catch (InterruptedException exception) { Thread.currentThread().interrupt(); } @@ -198,15 +202,15 @@ private Stream walksQueueConsumer( private static final class ExternalTerminationFlag implements TerminationFlag { private volatile boolean running = true; - private final TerminationFlag inner; + private final Algorithm algo; - ExternalTerminationFlag(TerminationFlag inner) { - this.inner = inner; + ExternalTerminationFlag(Algorithm algo) { + this.algo = algo; } @Override public boolean running() { - return this.running && this.inner.running(); + return this.running && this.algo.getTerminationFlag().running(); } void stop() { @@ -326,9 +330,13 @@ public void run() { private boolean flushBuffer(int bufferLength) { bufferLength = Math.min(bufferLength, this.buffer.length); - for (int i = 0; i < bufferLength && terminationFlag.running(); i++) { + int i = 0; + while (i < bufferLength && terminationFlag.running()) { try { - walks.put(this.buffer[i]); + // allow termination to occur if queue is full + if (walks.offer(this.buffer[i], 100, TimeUnit.MILLISECONDS)) { + i++; + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); return false; diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccStreamConfig.java b/algo/src/main/java/org/neo4j/gds/wcc/WccStreamConfig.java index 340d05d437d..51fda2ba8a3 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccStreamConfig.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccStreamConfig.java @@ -21,12 +21,19 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.config.CommunitySizeConfig; import org.neo4j.gds.core.CypherMapWrapper; +import java.util.Optional; + @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface WccStreamConfig extends WccBaseConfig { +public interface WccStreamConfig extends WccBaseConfig, CommunitySizeConfig { + + @Override + @Configuration.Key("minComponentSize") + Optional minCommunitySize(); static WccStreamConfig of(CypherMapWrapper userInput) { WccStreamConfigImpl wccStreamConfig = new WccStreamConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccWriteConfig.java b/algo/src/main/java/org/neo4j/gds/wcc/WccWriteConfig.java index 42e8ec9945c..b8eba23f7ee 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccWriteConfig.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccWriteConfig.java @@ -21,14 +21,20 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.ComponentSizeConfig; +import org.neo4j.gds.config.CommunitySizeConfig; import org.neo4j.gds.config.WritePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; +import java.util.Optional; + @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface WccWriteConfig extends WccBaseConfig, WritePropertyConfig, ComponentSizeConfig { +public interface WccWriteConfig extends WccBaseConfig, WritePropertyConfig, CommunitySizeConfig { + + @Override + @Configuration.Key("minComponentSize") + Optional minCommunitySize(); static WccWriteConfig of(CypherMapWrapper userInput) { return new WccWriteConfigImpl(userInput); diff --git a/algo/src/test/java/org/neo4j/gds/beta/indexInverse/InverseRelationshipsTest.java b/algo/src/test/java/org/neo4j/gds/beta/indexInverse/InverseRelationshipsTest.java index 1c75624deef..88ec03e6aac 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/indexInverse/InverseRelationshipsTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/indexInverse/InverseRelationshipsTest.java @@ -86,7 +86,7 @@ void shouldCreateIndexedRelationships(int concurrency) { // we need to use the same name for assertGraphEquals to work graphStore.deleteRelationships(relationshipType); - graphStore.addRelationshipType(relationshipType, inverseRelationshipsPerType.get(relationshipType)); + graphStore.addRelationshipType(inverseRelationshipsPerType.get(relationshipType)); for (String relationshipPropertyKey : inverseGraphStore.relationshipPropertyKeys()) { assertGraphEquals( @@ -118,7 +118,7 @@ void shouldIndexMultipleTypes(Object relTypes) { internalTypes.forEach(internalType -> { // we need to use the same name for assertGraphEquals to work graphStore.deleteRelationships(internalType); - graphStore.addRelationshipType(internalType, inverseRelationshipsPerType.get(internalType)); + graphStore.addRelationshipType(inverseRelationshipsPerType.get(internalType)); }); for (String relationshipPropertyKey : inverseGraphStore.relationshipPropertyKeys()) { diff --git a/algo/src/test/java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java b/algo/src/test/java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java index a3c233122c3..295eb7a5fb1 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java @@ -136,8 +136,8 @@ void testParallelK1Coloring() { .isLessThan(20L); assertThat(usedColors.size()) - .as("Used colors should be less than or equal to 20") - .isLessThanOrEqualTo(20); + .as("Used colors should be less than or equal to 21") + .isLessThanOrEqualTo(21); } diff --git a/algo/src/test/java/org/neo4j/gds/beta/undirected/ToUndirectedTest.java b/algo/src/test/java/org/neo4j/gds/beta/undirected/ToUndirectedTest.java index 7066237bd4d..43a57663caf 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/undirected/ToUndirectedTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/undirected/ToUndirectedTest.java @@ -82,7 +82,7 @@ void shouldCreateUndirectedRelationships(int concurrency) { Pools.DEFAULT ).compute(); - directedGraphStore.addRelationshipType(RelationshipType.of(config.mutateRelationshipType()), undirectedRelationships); + directedGraphStore.addRelationshipType(undirectedRelationships); for (String relationshipPropertyKey : undirectedGraphStore.relationshipPropertyKeys()) { assertGraphEquals( @@ -129,7 +129,7 @@ void shouldCreateUndirectedRelationshipsWithSingleRelationshipProperty(int concu Pools.DEFAULT ).compute(); - singleDirectedGraphStore.addRelationshipType(RelationshipType.of(config.mutateRelationshipType()), undirectedRelationships); + singleDirectedGraphStore.addRelationshipType(undirectedRelationships); assertGraphEquals( singleUndirectedGraphStore.getGraph(RelationshipType.of("T2"), Optional.of("prop1")), @@ -174,7 +174,7 @@ void shouldCreateUndirectedRelationshipsWithNoRelationshipProperty(int concurren Pools.DEFAULT ).compute(); - noPropertyDirectedGraphStore.addRelationshipType(RelationshipType.of(config.mutateRelationshipType()), undirectedRelationships); + noPropertyDirectedGraphStore.addRelationshipType(undirectedRelationships); assertGraphEquals( noPropertyUndirectedGraphStore.getGraph(RelationshipType.of("T2")), diff --git a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapseMultiPathsTest.java b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapseMultiPathsTest.java index fea823762b4..ae8d4f754ef 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapseMultiPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapseMultiPathsTest.java @@ -108,6 +108,7 @@ void shouldFollowRoutesFromMovies() { var path = new CollapsePath( pathTemplates, false, + RelationshipType.of("REL"), 2, Pools.DEFAULT @@ -170,6 +171,7 @@ void shouldTurnEveryRouteIntoRelationship() { new Graph[]{plane} ), false, + RelationshipType.of("REL"), 2, Pools.DEFAULT diff --git a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java index 3e7b0da30cb..c7499f19ff0 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java @@ -119,6 +119,7 @@ void testCreatingRelationships() { var relationships = new CollapsePath( Collections.singletonList(new Graph[]{tookRel, tookRel}), false, + RelationshipType.of("SAME_DRUG"), 2, Pools.DEFAULT @@ -134,6 +135,7 @@ void testAllowCreatingSelfLoops() { var relationships = new CollapsePath( Collections.singletonList(new Graph[]{tookRel, tookRel}), true, + RelationshipType.of("SAME_DRUG"), 2, Pools.DEFAULT @@ -147,6 +149,7 @@ void runWithDifferentRelationshipTypes() { var relationships = new CollapsePath( Collections.singletonList(new Graph[]{tookGraph, takenByGraph}), false, + RelationshipType.of("SAME_DRUG"), 2, Pools.DEFAULT ).compute(); @@ -155,7 +158,7 @@ void runWithDifferentRelationshipTypes() { } private void assertResultGraph(GraphStore graphStore, SingleTypeRelationships relationships, String expected) { - graphStore.addRelationshipType(RelationshipType.of("SAME_DRUG"), relationships); + graphStore.addRelationshipType(relationships); assertGraphEquals( fromGdl(expected), @@ -194,7 +197,7 @@ void shouldComputeForAllNodesWithoutNodeLabelsSpecified() { var relationships = new CollapsePathAlgorithmFactory() .build(graphStore, config, ProgressTracker.NULL_TRACKER) .compute(); - graphStore.addRelationshipType(mutateRelType, relationships); + graphStore.addRelationshipType(relationships); var resultGraph = graphStore.getGraph(mutateRelType); // then two relationships should be created @@ -219,7 +222,7 @@ void shouldComputeForSubsetOfNodesWithNodeLabelsSpecified() { var relationships = new CollapsePathAlgorithmFactory() .build(graphStore, config, ProgressTracker.NULL_TRACKER) .compute(); - graphStore.addRelationshipType(mutateRelType, relationships); + graphStore.addRelationshipType(relationships); var resultGraph = graphStore.getGraph(mutateRelType); // a single relationship is created (there is no Dog) diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java index ea8976c75e4..6c089f07b73 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java @@ -42,13 +42,13 @@ import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.ml.core.features.FeatureExtraction; import org.neo4j.gds.ml.core.features.FeatureExtractor; -import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import java.util.List; import java.util.Optional; @@ -624,6 +624,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { ); RelationshipsBuilder firstRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(firstIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); @@ -647,6 +648,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { ); RelationshipsBuilder secondRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(secondIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java index afb6bbd3373..145972a1572 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java @@ -26,10 +26,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.beta.generator.PropertyProducer; +import org.neo4j.gds.beta.generator.RandomGraphGenerator; +import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -77,6 +81,7 @@ class GraphSageModelTrainerTest { private Graph unweightedGraph; @Inject private Graph arrayGraph; + private HugeObjectArray features; private GraphSageTrainConfigImpl.Builder configBuilder; @@ -97,6 +102,53 @@ void setUp() { .embeddingDimension(EMBEDDING_DIMENSION); } + // This reproduced bug in https://trello.com/c/BQ3e12K3/7826-250-graphsage-returns-nan-when-using-relationship-weights + // https://github.com/neo4j/graph-data-science/issues/250 + @ParameterizedTest + @EnumSource(AggregatorType.class) + void trainsWithRelationshipWeight(AggregatorType aggregatorType) { + + var config = GraphSageTrainConfigImpl.builder() + .randomSeed(42L) + .batchSize(100) + .relationshipWeightProperty("p") + .embeddingDimension(2) + .aggregator(aggregatorType) + .activationFunction(ActivationFunction.SIGMOID) + .featureProperties(List.of("features")) + .modelName("model") + .modelUser("") + .build(); + + var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + + int nodeCount = 5_000; + var bigGraph = RandomGraphGenerator + .builder() + .nodeCount(nodeCount) + .averageDegree(1) + .relationshipDistribution(RelationshipDistribution.UNIFORM) + .nodePropertyProducer(PropertyProducer.randomEmbedding("features", 1, -100, 100)) + .relationshipPropertyProducer(PropertyProducer.fixedDouble("p", 0.5)) + .seed(42L) + .build() + .generate(); + + features = HugeObjectArray.newArray(double[].class, nodeCount); + + LongStream.range(0, nodeCount).forEach(n -> features.set(n, bigGraph.nodeProperties().get("features").doubleArrayValue(n))); + + GraphSageModelTrainer.ModelTrainResult result = trainModel.train( + bigGraph, + features + ); + + assertThat(result.layers()) + .allSatisfy(layer -> assertThat(layer.weights()) + .noneMatch(weights -> TensorTestUtils.containsNaN(weights.data())) + ); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void trainsWithMeanAggregator(boolean useRelationshipWeight) { @@ -236,18 +288,7 @@ void testLosses() { assertThat(metrics.ranIterationsPerEpoch()).containsExactly(100, 100, 100, 100, 100, 100, 100, 100, 100, 100); assertThat(metrics.epochLosses().stream().mapToDouble(Double::doubleValue).toArray()) - .contains(new double[]{ - 18.25, - 16.31, - 16.41, - 16.21, - 14.96, - 14.97, - 14.31, - 16.17, - 14.90, - 15.58 - }, Offset.offset(0.05) + .contains(new double[]{19.55, 21.24, 19.90, 19.42, 17.87, 17.03, 17.04, 20.42, 15.86, 20.56}, Offset.offset(0.05) ); } @@ -280,18 +321,7 @@ void testLossesWithPoolAggregator() { assertThat(metrics.ranIterationsPerEpoch()).containsOnly(10); assertThat(metrics.epochLosses().stream().mapToDouble(Double::doubleValue).toArray()) - .contains(new double[]{ - 23.41, - 19.94, - 19.70, - 21.62, - 19.06, - 24.11, - 19.72, - 16.47, - 19.74, - 20.97 - }, Offset.offset(0.05) + .contains(new double[]{19.73, 21.25, 20.81, 23.13, 19.70, 25.34, 20.65, 17.10, 20.48, 21.51}, Offset.offset(0.05) ); } diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java index a15ed0fa801..35eb90754a2 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java @@ -117,7 +117,6 @@ void setUp() { graphStore = CSRGraphStoreUtil.createFromGraph( DatabaseId.random(), randomGraph, - "REL", Optional.of("weight"), 4 ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactoryTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactoryTest.java index 82ac56df6b1..b455e4f9512 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactoryTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactoryTest.java @@ -499,21 +499,21 @@ void testLogging() { "GraphSageTrain :: Train model :: Start", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Start", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 :: Start", - "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 :: Average loss per node: 26.49", + "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 :: Average loss per node: 25.58", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 100%", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 :: Start", - "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 :: Average loss per node: 25.58", + "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 :: Average loss per node: 26.69", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 100%", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Start", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 :: Start", - "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 :: Average loss per node: 25.28", + "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 :: Average loss per node: 23.29", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 100%", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 :: Start", - "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 :: Average loss per node: 25.23", + "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 :: Average loss per node: 22.88", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 100%", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Finished", diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java index 97686dd1295..32d8e88339f 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java @@ -28,12 +28,14 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.ResourceUtil; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.HugeSparseLongArray; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; @@ -45,11 +47,11 @@ import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; -import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import java.util.List; import java.util.Map; @@ -311,6 +313,37 @@ void shouldEstimateMemory( ); } + @Test + void estimationShouldUseGeneratedDimensionIfOutputIsMissing() { + var inputDimension = 1000L; + var inputRatio = 0.1; + var graphDims = GraphDimensions.of((long) 1e6); + var concurrency = 4; + + var bigEstimation = new HashGNNFactory<>() + .memoryEstimation(HashGNNStreamConfigImpl + .builder() + .generateFeatures(Map.of("dimension", inputDimension, "densityLevel", 1)) + .iterations(3) + .embeddingDensity(100) + .build()) + .estimate(graphDims, concurrency) + .memoryUsage(); + + var smallEstimation = new HashGNNFactory<>() + .memoryEstimation(HashGNNStreamConfigImpl + .builder() + .generateFeatures(Map.of("dimension", (long) (inputRatio * inputDimension), "densityLevel", 1)) + .iterations(3) + .embeddingDensity(100) + .build()) + .estimate(graphDims, concurrency) + .memoryUsage(); + + var outputRatio = (double) smallEstimation.min / bigEstimation.min; + assertThat(outputRatio).isCloseTo(inputRatio, Offset.offset(0.1)); + } + @ParameterizedTest @CsvSource(value = {"true", "false"}) void shouldLogProgress(boolean dense) { @@ -372,6 +405,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { ); RelationshipsBuilder firstRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(firstIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); @@ -395,6 +429,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { ); RelationshipsBuilder secondRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(secondIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java index 6e98729e9fb..06712e2221c 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java @@ -36,6 +36,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.api.Graph; @@ -53,10 +54,10 @@ import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import org.neo4j.gds.embeddings.node2vec.Node2VecBaseConfig.EmbeddingInitializer; import org.neo4j.gds.gdl.GdlFactory; import org.neo4j.gds.ml.core.tensor.FloatVector; -import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import java.util.List; import java.util.Optional; @@ -277,6 +278,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit ); RelationshipsBuilder firstRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(firstIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); @@ -298,6 +300,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit ); RelationshipsBuilder secondRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(secondIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java b/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java index 3d38d71303a..317fc5cbe52 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java @@ -101,8 +101,8 @@ void testGraphAggregation() { assertGraphEquals( fromGdl( "(c1), (c2), " + - "(c1)-[:REL {w: 4.0}]->(c2), " + - "(c2)-[:REL {w: 4.0}]->(c1)" + "(c1)-[:_IGNORED_ {w: 4.0}]->(c2), " + + "(c2)-[:_IGNORED_ {w: 4.0}]->(c1)" ), aggregatedGraph ); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java b/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java index 193a1674ea1..7e040ee894e 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java @@ -41,6 +41,7 @@ import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.extension.TestGraph; +import org.neo4j.gds.gdl.GdlFactory; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -154,6 +155,8 @@ void shouldWorkWithBestSeed() { community -> assertThat(community).containsExactlyInAnyOrder("a1", "a5", "a6", "a7") ); assertThat(communitiesMap.keySet()).containsExactly(4000L, 5000L); + + assertThat(leidenResult.modularity()).isGreaterThan(0); } @Test @@ -364,4 +367,29 @@ void shouldLogProgress() { "Leiden :: Finished" ); } + + @Test + void shouldWorkIfStopAtFirstIteration() { + + var query = " CREATE\n" + + " (c:C {id: 0}) \n" + + " , (d:D {id: 1}) "; + var empty = GdlFactory.of(query).build().getUnion(); + var result = new Leiden( + empty, + 5, + 1, + 0.01, + false, + 42, + null, + 0.1, + 1, + ProgressTracker.NULL_TRACKER + ).compute(); + var communities = result.communities(); + assertThat(communities.toArray()).isEqualTo(new long[]{0, 1}); + + + } } diff --git a/algo/src/test/java/org/neo4j/gds/leiden/LeidenWeightedCliqueTest.java b/algo/src/test/java/org/neo4j/gds/leiden/LeidenWeightedCliqueTest.java new file mode 100644 index 00000000000..96d93d6be1d --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/leiden/LeidenWeightedCliqueTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.leiden; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.Orientation; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.IdFunction; +import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.extension.TestGraph; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.core.ProcedureConstants.TOLERANCE_DEFAULT; + +@GdlExtension +class LeidenWeightedCliqueTest { + + @GdlGraph(orientation = Orientation.UNDIRECTED) + private static final String DB_CYPHER = + "CREATE" + + "(a : Node), "+ + "(b : Node), "+ + "(c : Node), "+ + "(d : Node), "+ + "(e : Node), "+ + + "(a)-[:R {w: 0.1}]->(b),"+ + "(a)-[:R {w: 0.1}]->(c),"+ + "(a)-[:R {w: 0.1}]->(d),"+ + "(a)-[:R {w: 0.1}]->(e),"+ + + "(b)-[:R {w: 0.1}]->(c),"+ + "(b)-[:R {w: 0.1}]->(d),"+ + "(b)-[:R {w: 0.1}]->(e),"+ + + "(c)-[:R {w: 0.1}]->(d),"+ + "(c)-[:R {w: 0.1}]->(e),"+ + + "(d)-[:R {w: 0.1}]->(e)"; + + @Inject + private TestGraph graph; + + @Inject + private IdFunction idFunction; + + @Test + void weightedLeiden() { + var maxLevels = 10; + + Leiden leiden = new Leiden( + graph, + maxLevels, + 1.0, + 0.01, + true, + 19L, + null, + TOLERANCE_DEFAULT, + 4, + ProgressTracker.NULL_TRACKER + + ); + var leidenResult = leiden.compute(); + assertThat(Arrays.stream(leidenResult.communities().toArray()).distinct().count()).isEqualTo(1); + + } + +} diff --git a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java index 9bb46d93aa6..00e04bd2630 100644 --- a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java +++ b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.louvain; +import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -27,9 +28,12 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.config.RandomGraphGeneratorConfig; +import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; import org.neo4j.gds.core.concurrency.Pools; @@ -45,11 +49,14 @@ import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.modularity.ModularityCalculator; import java.util.Map; import java.util.Optional; +import java.util.function.LongUnaryOperator; import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -516,6 +523,39 @@ void shouldThrowOnNegativeSeed() { assertThatThrownBy(algorithm::compute).hasMessageContaining("non-negative"); + } + + @Test + void shouldGiveSameResultWithCalculator() { + var myGraph = RandomGraphGenerator + .builder() + .nodeCount(1_000) + .averageDegree(10) + .relationshipDistribution(RelationshipDistribution.UNIFORM) + .direction(Direction.UNDIRECTED) + .allowSelfLoops(RandomGraphGeneratorConfig.AllowSelfLoops.YES) + .aggregation(Aggregation.SINGLE) + .seed(42) + .build() + .generate(); + + var louvain = new Louvain( + myGraph, + ImmutableLouvainStreamConfig.builder().build(), + false, + 10, + 10, + TOLERANCE_DEFAULT, + 4, + ProgressTracker.NULL_TRACKER, + Pools.DEFAULT + ); + var result = louvain.compute(); + assertThat(louvain.levels()).isGreaterThan(1); + LongUnaryOperator vToCommunity = v -> result.getCommunity(v); + var modularityCalculator = ModularityCalculator.create(myGraph, vToCommunity, 4); + double calculatedModularity = modularityCalculator.compute().totalModularity(); + assertThat(result.modularities()[louvain.levels() - 1]).isCloseTo(calculatedModularity, Offset.offset(1e-5)); } } diff --git a/algo/src/test/java/org/neo4j/gds/modularity/ModularityCalculatorTest.java b/algo/src/test/java/org/neo4j/gds/modularity/ModularityCalculatorTest.java index 1ab530f984b..eeefba3ecbc 100644 --- a/algo/src/test/java/org/neo4j/gds/modularity/ModularityCalculatorTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularity/ModularityCalculatorTest.java @@ -19,7 +19,8 @@ */ package org.neo4j.gds.modularity; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.extension.GdlExtension; @@ -35,12 +36,12 @@ class ModularityCalculatorTest { @GdlGraph(orientation = Orientation.UNDIRECTED) static final String GRAPH = "CREATE " + - " (a1: Node { communityId: 0 })," + - " (a2: Node { communityId: 0 })," + - " (a3: Node { communityId: 5 })," + - " (a4: Node { communityId: 0 })," + - " (a5: Node { communityId: 5 })," + - " (a6: Node { communityId: 5 })," + + " (a1: Node { communityId: 0, communityId2: 200 })," + + " (a2: Node { communityId: 0, communityId2: 200 })," + + " (a3: Node { communityId: 5, communityId2: 205 })," + + " (a4: Node { communityId: 0, communityId2: 200 })," + + " (a5: Node { communityId: 5, communityId2: 205 })," + + " (a6: Node { communityId: 5, communityId2: 205 })," + " (a1)-[:R]->(a2)," + " (a1)-[:R]->(a4)," + @@ -56,11 +57,12 @@ class ModularityCalculatorTest { @Inject private GraphStore graphStore; - @Test - void compute() { - var modularityCalculator = new ModularityCalculator( + @ParameterizedTest + @CsvSource({"communityId,0", "communityId2,200"}) + void compute(String communityId, int startingId) { + var modularityCalculator = ModularityCalculator.create( graph, - graphStore.nodeProperty("communityId").values()::longValue, + graphStore.nodeProperty(communityId).values()::longValue, 4 ); @@ -74,8 +76,9 @@ void compute() { assertThat(modularities) .containsExactlyInAnyOrder( - CommunityModularity.of(0L, community_0_score), - CommunityModularity.of(5L, community_5_score) + CommunityModularity.of(startingId, community_0_score), + CommunityModularity.of(startingId + 5L, community_5_score) ); } + } diff --git a/algo/src/test/java/org/neo4j/gds/modularity/RelationshipCountCollectorTest.java b/algo/src/test/java/org/neo4j/gds/modularity/RelationshipCountCollectorTest.java index aeda650ca7b..ca4bf49c28b 100644 --- a/algo/src/test/java/org/neo4j/gds/modularity/RelationshipCountCollectorTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularity/RelationshipCountCollectorTest.java @@ -21,7 +21,6 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeAtomicDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.partition.Partition; @@ -63,7 +62,6 @@ class RelationshipCountCollectorTest { void collect() { var insideRelationships = HugeAtomicDoubleArray.newArray(graph.nodeCount()); var totalCommunityRelationships = HugeAtomicDoubleArray.newArray(graph.nodeCount()); - var communityTracker = HugeAtomicBitSet.create(graph.nodeCount()); var communities = HugeLongArray.of(0, 0, 5, 0, 5, 5); var totalRelationshipWeight = new DoubleAdder(); @@ -72,7 +70,6 @@ void collect() { graph, insideRelationships, totalCommunityRelationships, - communityTracker, communities::get, totalRelationshipWeight ).run(); diff --git a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java index 0b7b3569cb0..169ffa7cc43 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.paths.delta; +import org.assertj.core.data.Offset; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -29,6 +30,10 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.beta.generator.PropertyProducer; +import org.neo4j.gds.beta.generator.RandomGraphGeneratorBuilder; +import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.concurrency.Pools; @@ -39,8 +44,10 @@ import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.paths.delta.config.ImmutableAllShortestPathsDeltaStreamConfig; +import org.neo4j.gds.paths.dijkstra.Dijkstra; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.DoubleStream; @@ -337,4 +344,60 @@ void singleSource(double delta, int concurrency, long idOffset) { } private DeltaSteppingTest() {} + + @Test + void shouldGiveSameResultsAsDijkstra() { + int nodeCount = 3_000; + long seed = 42L; + long start = 42; + int concurrency = 4; + var newGraph = new RandomGraphGeneratorBuilder() + .direction(Direction.DIRECTED) + .averageDegree(10) + .relationshipDistribution(RelationshipDistribution.POWER_LAW) + .relationshipPropertyProducer(PropertyProducer.randomDouble("foo", 1, 10)) + .nodeCount(nodeCount) + .seed(seed) + .build() + .generate(); + + var config = ImmutableAllShortestPathsDeltaStreamConfig.builder() + .concurrency(concurrency) + .sourceNode(start) + .trackRelationships(true) + .build(); + var deltaStepping = DeltaStepping.of( + newGraph, + config, + Pools.DEFAULT, + ProgressTracker.NULL_TRACKER + ).compute(); + + var dijkstraAlgo = Dijkstra + .singleSource(newGraph, config, Optional.empty(), ProgressTracker.NULL_TRACKER) + .compute(); + + double[] delta = new double[nodeCount]; + double[] djikstra = new double[nodeCount]; + + double deltaSum = 0; + double dijkstraSum = 0; + + for (var path : deltaStepping.pathSet()) { + delta[(int) path.targetNode()] = path.totalCost(); + } + + for (var path : dijkstraAlgo.pathSet()) { + djikstra[(int) path.targetNode()] = path.totalCost(); + } + for (int i = 0; i < nodeCount; ++i) { + deltaSum += delta[i]; + dijkstraSum += djikstra[i]; + assertThat(djikstra[i]).isCloseTo(delta[i], Offset.offset(1e-5)); + + } + assertThat(deltaSum).isCloseTo(dijkstraSum, Offset.offset(1e-5)); + + } + } diff --git a/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java b/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java index c3af4a3b569..6275662143e 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java @@ -32,6 +32,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -85,7 +86,7 @@ void shouldComputeMemoryEstimation(int nodeCount, long expectedBytes) { } // https://en.wikipedia.org/wiki/Yen%27s_algorithm#/media/File:Yen's_K-Shortest_Path_Algorithm,_K=3,_A_to_F.gif - @GdlGraph + @GdlGraph(aggregation = Aggregation.SINGLE) private static final String DB_CYPHER = "CREATE" + " (c:C {id: 0})" + @@ -165,7 +166,7 @@ static Stream> pathInput() { @ParameterizedTest @MethodSource("pathInput") void compute(Collection expectedPaths) { - assertResult(graph, idFunction, expectedPaths); + assertResult(graph, idFunction, expectedPaths, false); } @Test @@ -238,8 +239,13 @@ void shouldCloseProgressTasksOnEmptyResult() { ); } - private static void assertResult(Graph graph, IdFunction idFunction, Collection expectedPaths) { - var expectedPathResults = expectedPathResults(idFunction, expectedPaths); + private static void assertResult( + Graph graph, + IdFunction idFunction, + Collection expectedPaths, + boolean trackRelationships + ) { + var expectedPathResults = expectedPathResults(idFunction, expectedPaths, trackRelationships); var firstResult = expectedPathResults .stream() @@ -267,7 +273,11 @@ private static void assertResult(Graph graph, IdFunction idFunction, Collection< } @NotNull - private static Set expectedPathResults(IdFunction idFunction, Collection expectedPaths) { + private static Set expectedPathResults( + IdFunction idFunction, + Collection expectedPaths, + boolean trackRelationships + ) { var index = new MutableInt(0); return expectedPaths.stream() .map(expectedPath -> new GDLHandler.Builder() @@ -312,7 +322,7 @@ private static Set expectedPathResults(IdFunction idFunction, Collec .sourceNode(sourceNode.getId()) .targetNode(targetNode.getId()) .nodeIds(nodeIds) - .relationshipIds(relationshipIds) + .relationshipIds(trackRelationships ? relationshipIds : new long[0]) .costs(costs) .build(); }) @@ -375,7 +385,7 @@ Stream> pathInput() { @ParameterizedTest @MethodSource("pathInput") void compute(Collection expectedPaths) { - assertResult(graph, idFunction, expectedPaths); + assertResult(graph, idFunction, expectedPaths, true); } } } diff --git a/algo/src/test/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityTest.java index 83d5eb62b87..093f5b7e549 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityTest.java @@ -20,6 +20,12 @@ package org.neo4j.gds.similarity.filterednodesim; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.TestProgressTracker; +import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -30,6 +36,8 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.assertj.Extractors.removingThreadId; +import static org.neo4j.gds.compat.TestLog.INFO; @GdlExtension class FilteredNodeSimilarityTest { @@ -154,4 +162,55 @@ void shouldSurviveIoannisFurtherObjections() { nodeSimilarity.release(); } + + @ParameterizedTest + @ValueSource(ints = {1, 2}) + void shouldLogProgressAccurately(int concurrency) { + var sourceNodeFilter = List.of(2L, 3L); + + var config = ImmutableFilteredNodeSimilarityStreamConfig.builder() + .sourceNodeFilter(NodeFilterSpecFactory.create(sourceNodeFilter)) + .concurrency(concurrency) + .topK(0) + .topN(10) + .build(); + var progressTask = new FilteredNodeSimilarityFactory<>().progressTask(graph, config); + TestLog log = Neo4jProxy.testLog(); + var progressTracker = new TestProgressTracker( + progressTask, + log, + concurrency, + EmptyTaskRegistryFactory.INSTANCE + ); + + + new FilteredNodeSimilarityFactory<>().build( + graph, + config, + progressTracker + ).compute(); + + + assertThat(log.getMessages(INFO)) + .extracting(removingThreadId()) + .containsExactly( + "FilteredNodeSimilarity :: Start", + "FilteredNodeSimilarity :: prepare :: Start", + "FilteredNodeSimilarity :: prepare 33%", + "FilteredNodeSimilarity :: prepare 55%", + "FilteredNodeSimilarity :: prepare 66%", + "FilteredNodeSimilarity :: prepare 100%", + "FilteredNodeSimilarity :: prepare :: Finished", + "FilteredNodeSimilarity :: compare node pairs :: Start", + "FilteredNodeSimilarity :: compare node pairs 12%", + "FilteredNodeSimilarity :: compare node pairs 25%", + "FilteredNodeSimilarity :: compare node pairs 37%", + "FilteredNodeSimilarity :: compare node pairs 50%", + "FilteredNodeSimilarity :: compare node pairs 62%", + "FilteredNodeSimilarity :: compare node pairs 75%", + "FilteredNodeSimilarity :: compare node pairs 100%", + "FilteredNodeSimilarity :: compare node pairs :: Finished", + "FilteredNodeSimilarity :: Finished" + ); + } } diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java index 1b2e4b4b638..656c25f8541 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java @@ -23,8 +23,8 @@ import net.jqwik.api.From; import net.jqwik.api.Property; import org.eclipse.collections.api.tuple.primitive.IntIntPair; -import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.core.huge.DirectIdMap; +import org.neo4j.gds.core.utils.IdentityPropertyValues; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.partition.Partition; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -51,17 +51,7 @@ void neighborsForKEqualsNMinus1startWithEachOtherAsNeighbors( nodeCount ); - var nodeProperties = new LongNodePropertyValues() { - @Override - public long longValue(long nodeId) { - return nodeId; - } - - @Override - public long size() { - return nodeCount; - } - }; + var nodeProperties = new IdentityPropertyValues(nodeCount); var similarityComputer = SimilarityComputer.ofProperty( idMap, diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java new file mode 100644 index 00000000000..4c2bfe93969 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.knn.metrics; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; + +import static org.assertj.core.api.Assertions.assertThat; + +class LongArrayPropertySimilarityComputerTest { + + @Test + void usingSparseProperties() { + long[][] inputValues = {{42, 24}, null, {84, 48}}; + + var sparseProperties = new LongArrayNodePropertyValues() { + @Override + public long[] longArrayValue(long nodeId) { + return inputValues[(int) nodeId]; + } + + @Override + public long nodeCount() { + return inputValues.length; + } + }; + + var sortedValues = new LongArrayPropertySimilarityComputer.SortedLongArrayPropertyValues(sparseProperties); + + assertThat(sortedValues.longArrayValue(0)).containsExactly(24, 42); + assertThat(sortedValues.longArrayValue(1)).isNull(); + assertThat(sortedValues.longArrayValue(2)).containsExactly(48, 84); + } + +} diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java index 7bec193e48a..33359d3a7a6 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java @@ -265,7 +265,7 @@ public long[] longArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return nodeCount; } }; diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java index 079603b904c..a07f3c10711 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java @@ -65,7 +65,7 @@ void shouldTerminate() { db, nodeSimilarity, nhs -> nodeSimilarity.computeToStream(), - 100 + 200 ); } diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index f3c965233aa..7bdfea09fbb 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -773,6 +773,29 @@ void shouldComputeMemrecWithTop(int topK) { assertEquals(expected.memoryUsage(), actual.memoryUsage()); } + @Test + void shouldComputeMemrecWithTopKAndTopNGreaterThanNodeCount() { + GraphDimensions dimensions = ImmutableGraphDimensions.builder() + .nodeCount(100) + .relCountUpperBound(20_000) + .build(); + + NodeSimilarityWriteConfig config = ImmutableNodeSimilarityWriteConfig + .builder() + .similarityCutoff(0.0) + .topK(Integer.MAX_VALUE) + .topN(Integer.MAX_VALUE) + .writeProperty("writeProperty") + .writeRelationshipType("writeRelationshipType") + .build(); + + MemoryTree actual = new NodeSimilarityFactory<>().memoryEstimation(config).estimate(dimensions, 1); + + assertEquals(570592, actual.memoryUsage().min); + assertEquals(732192, actual.memoryUsage().max); + + } + @ParameterizedTest(name = "topK = {0}, concurrency = {1}") @MethodSource("topKAndConcurrencies") void shouldLogMessages(int topK, int concurrency) { @@ -799,6 +822,35 @@ void shouldLogMessages(int topK, int concurrency) { ); } + @ParameterizedTest(name = "topK = {0}, concurrency = {1}") + @MethodSource("topKAndConcurrencies") + void shouldNotLogMessagesWhenLoggingIsDisabled(int topK, int concurrency) { + var graph = naturalGraph; + var config = configBuilder().topN(100).topK(topK).concurrency(concurrency).logProgress(false).build(); + + var progressLog = Neo4jProxy.testLog(); + var nodeSimilarity = new NodeSimilarityFactory<>().build( + graph, + config, + progressLog, + EmptyTaskRegistryFactory.INSTANCE + ); + + nodeSimilarity.compute(); + + assertThat(progressLog.getMessages(INFO)) + .as("When progress logging is disabled we only log `start` and `finished`.") + .extracting(removingThreadId()) + .containsExactly( + "NodeSimilarity :: Start", + "NodeSimilarity :: prepare :: Start", + "NodeSimilarity :: prepare :: Finished", + "NodeSimilarity :: compare node pairs :: Start", + "NodeSimilarity :: compare node pairs :: Finished", + "NodeSimilarity :: Finished" + ); + } + @ParameterizedTest(name = "concurrency = {0}") @ValueSource(ints = {1,2}) void shouldLogProgress(int concurrency) { diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java new file mode 100644 index 00000000000..0c4ab250cad --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.nodesim; + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import static org.assertj.core.api.Assertions.assertThat; + +@GdlExtension + class UnionGraphWeightedNodeSimilarityTest { + + @GdlGraph + private static final String DB_CYPHER = + "CREATE " + + " (a1: A)," + + " (a2: A)," + + " (b1: B)," + + " (b2: B)," + + " (b3: B)," + + "(a1)-[:R1 {w: 5}]->(b1),"+ + "(a1)-[:R2 {w: 10}]->(b2),"+ + "(a2)-[:R2 {w: 10}]->(b2),"+ + "(a2)-[:R1 {w: 8}]->(b3)"; + + @Inject + private Graph graph; + + @Test + void shouldWorkWithUnionGraph(){ + + var config = NodeSimilarityStreamConfigImpl.builder() + .relationshipWeightProperty("w") + .concurrency(1) + .topN(1) + .build(); + + var nodeSimilarity = NodeSimilarity.create(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var result = nodeSimilarity.compute().streamResult().findFirst().get(); + + //input should be (0 + 10 + 0)/ (5 + 10 + 8) = 10/23 + assertThat(result.similarity).isCloseTo((10) / (23.0), Offset.offset(1E-5)); + } +} diff --git a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java index 47ad8d3fda9..8bf408cc79c 100644 --- a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java +++ b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java @@ -36,6 +36,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -411,6 +412,37 @@ void testWithConfiguredOffsetStartNodes() { .anyMatch(walk -> walk[0] == bInternalId); } + /** + * Ensure that when termination flag is set externally, we terminate the walk + */ + @Test + void testSetTerminationFlagAndMultipleRuns() { + for (int i = 0; i < 3; i++) { + Node2VecStreamConfig config = ImmutableNode2VecStreamConfig.builder() + .walkBufferSize(1) + .build(); + + var randomWalk = RandomWalk.create( + graph, + config, + ProgressTracker.NULL_TRACKER, + Pools.DEFAULT + ); + + var stream = randomWalk.compute(); + long count = stream.limit(10).count(); + + randomWalk.setTerminationFlag(new TerminationFlag() { + @Override + public boolean running() { + return false; + } + }); + + assertEquals(10, count); + } + } + @Nested class ProgressTracking { diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index b74a506f5f9..59f1eb06e62 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -19,8 +19,11 @@ */ package org.neo4j.gds.impl.spanningtree; +import com.carrotsearch.hppc.BitSet; +import org.jetbrains.annotations.NotNull; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.queue.HugeLongPriorityQueue; @@ -45,8 +48,6 @@ public class KSpanningTree extends Algorithm { private final long startNodeId; private final long k; - private SpanningTree spanningTree; - public KSpanningTree( Graph graph, DoubleUnaryOperator minMax, @@ -71,38 +72,248 @@ public SpanningTree compute() { startNodeId, progressTracker ); + prim.setTerminationFlag(getTerminationFlag()); SpanningTree spanningTree = prim.compute(); - HugeLongArray parent = spanningTree.parentArray(); - long parentSize = parent.size(); - HugeLongPriorityQueue priorityQueue = minMax == Prim.MAX_OPERATOR ? HugeLongPriorityQueue.min(parentSize) : HugeLongPriorityQueue.max( - parentSize); - progressTracker.beginSubTask(parentSize); - for (long i = 0; i < parentSize && terminationFlag.running(); i++) { - long p = parent.get(i); - if (p == -1) { - continue; - } - priorityQueue.add(i, graph.relationshipProperty(p, i, 0.0D)); - progressTracker.logProgress(); - } + + var outputTree = growApproach(spanningTree); progressTracker.endSubTask(); - progressTracker.beginSubTask(k - 1); - // remove k-1 relationships - for (long i = 0; i < k - 1 && terminationFlag.running(); i++) { - long cutNode = priorityQueue.pop(); - parent.set(cutNode, -1); - progressTracker.logProgress(); + return outputTree; + } + + @NotNull + private HugeLongPriorityQueue createPriorityQueue(long parentSize, boolean pruning) { + boolean minQueue = minMax == Prim.MIN_OPERATOR; + //if pruning, we remove the worst (max if it's a minimization problem) + //therefore we flip the priority queue + if (pruning) { + minQueue = !minQueue; } - progressTracker.endSubTask(); - this.spanningTree = prim.getSpanningTree(); - progressTracker.endSubTask(); - return this.spanningTree; + HugeLongPriorityQueue priorityQueue = minQueue + ? HugeLongPriorityQueue.min(parentSize) + : HugeLongPriorityQueue.max(parentSize); + return priorityQueue; } @Override public void release() { graph = null; - spanningTree = null; } + + private double init(HugeLongArray parent, HugeDoubleArray costToParent, SpanningTree spanningTree) { + graph.forEachNode((nodeId) -> { + parent.set(nodeId, spanningTree.parent(nodeId)); + costToParent.set(nodeId, spanningTree.costToParent(nodeId)); + return true; + }); + return spanningTree.totalWeight(); + } + + + private SpanningTree growApproach(SpanningTree spanningTree) { + + //this approach grows gradually the MST found in the previous step + //when it is about to get larger than K, we crop the current worst leaf if the new value to be added + // is actually better + if (spanningTree.effectiveNodeCount() < k) + return spanningTree; + + HugeLongArray outDegree = HugeLongArray.newArray(graph.nodeCount()); + + HugeLongArray parent = HugeLongArray.newArray(graph.nodeCount()); + HugeDoubleArray costToParent = HugeDoubleArray.newArray(graph.nodeCount()); + + init(parent, costToParent, spanningTree); + double totalCost = 0; + var priorityQueue = createPriorityQueue(graph.nodeCount(), false); + var toTrim = createPriorityQueue(graph.nodeCount(), true); + + //priority-queue does not have a remove method + // so we need something to know if a node is still a leaf or not + BitSet exterior = new BitSet(graph.nodeCount()); + //at any point, the tree has a root we mark its neighbors in this bitset to avoid looping to find them + BitSet rootNodeAdjacent = new BitSet(graph.nodeCount()); + //we just save which nodes are in the final output and not (just to do clean-up; probably can be avoided) + BitSet included = new BitSet(graph.nodeCount()); + + priorityQueue.add(startNodeId, 0); + long root = startNodeId; //current root is startNodeId + long nodesInTree = 0; + progressTracker.beginSubTask(graph.nodeCount()); + while (!priorityQueue.isEmpty()) { + long node = priorityQueue.top(); + progressTracker.logProgress(); + double associatedCost = priorityQueue.cost(node); + priorityQueue.pop(); + long nodeParent = parent.get(node); + + boolean nodeAdded = false; + if (nodesInTree < k) { //if we are smaller, we can just add it no problemo + nodesInTree++; + nodeAdded = true; + } else { + var nodeToTrim = findNextValidLeaf(toTrim, exterior); //a leaf node with currently theworst cost + if (parent.get(node) == nodeToTrim) { + //we cannot add it, if we're supposed to remove its parent + //TODO: should be totally feasible to consider the 2nd worst then. + continue; + } + + boolean shouldMove = moveMakesSense(associatedCost, toTrim.cost(nodeToTrim), minMax); + + if (shouldMove) { + nodeAdded = true; + + double value = toTrim.cost(nodeToTrim); + toTrim.pop(); + + long parentOfTrimmed = parent.get(nodeToTrim); + included.clear(nodeToTrim); //nodeToTrim is removed from the answer + clearNode(nodeToTrim, parent, costToParent); + totalCost -= value; //as well as its cost from the solution + + if (root != nodeToTrim) { //we are not removing the actual root + //reduce degree of parent + outDegree.set(parentOfTrimmed, outDegree.get(parentOfTrimmed) - 1); + long affectedNode = -1; + double affectedCost = -1; + long parentOutDegree = outDegree.get(parentOfTrimmed); + if (parentOfTrimmed == root) { //if its parent is the root + rootNodeAdjacent.clear(nodeToTrim); //remove the trimmed child + if (parentOutDegree == 1) { //root becomes a leaf + assert rootNodeAdjacent.cardinality() == 1; + //get the single sole child of root + var rootChild = rootNodeAdjacent.nextSetBit(0); + affectedNode = root; + affectedCost = costToParent.get(rootChild); + } + } else { + if (parentOutDegree == 0) { //if parent becomes a leaf + affectedNode = parentOfTrimmed; + affectedCost = costToParent.get(parentOfTrimmed); + } + } + if (affectedNode != -1) { //if a node has been converted to a leaf + updateExterior(affectedNode, affectedCost, toTrim, exterior); + } + } else { + //the root is removed, long live the new root! + assert rootNodeAdjacent.cardinality() == 1; + //the new root is the single sole child of old root + var newRoot = rootNodeAdjacent.nextSetBit(0); + rootNodeAdjacent.clear(); //empty everything + //find the children of the new root (this can happen once per node) + + fillChildren(newRoot, rootNodeAdjacent, parent, included); + + root = newRoot; + //set it as root + clearNode(root, parent, costToParent); + //check if root is a degree-1 to add to exterior + if (outDegree.get(root) == 1) { + //get single child + var rootChild = rootNodeAdjacent.nextSetBit(0); + priorityQueue.add(root, costToParent.get(rootChild)); + exterior.set(root); + } + } + } + } + if (nodeAdded) { + included.set(node); // include it in the solution (for now!) + totalCost += associatedCost; //add its associated cost to the weight of tree + if (nodeParent == root) { //if it's parent is the root, update the bitset + rootNodeAdjacent.set(node); + } + if (node != root) { //this only happens for startNode to be fair + //the node's parent gets an update in degree + outDegree.set(nodeParent, outDegree.get(nodeParent) + 1); + exterior.clear(nodeParent); //and remoed from exterior if included + } + //then the node (being a leaf) is added to the trimming priority queu + toTrim.add(node, associatedCost); + exterior.set(node); //and the exterior + relaxNode(node, priorityQueue, parent, spanningTree); + + } else { + clearNode(node, parent, costToParent); + } + } + //post-processing step: anything not touched is reset to -1 + pruneUntouchedNodes(parent, costToParent, included); + progressTracker.endSubTask(); + return new SpanningTree(root, graph.nodeCount(), k, parent, costToParent, totalCost); + + } + + private void pruneUntouchedNodes(HugeLongArray parent, HugeDoubleArray costToParent, BitSet included) { + graph.forEachNode(nodeId -> { + if (!included.get(nodeId)) { + clearNode(nodeId, parent, costToParent); + } + return true; + }); + } + + private void clearNode(long node, HugeLongArray parent, HugeDoubleArray costToParent) { + parent.set(node, -1); + costToParent.set(node, -1); + } + + private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator minMax) { + if (minMax == Prim.MAX_OPERATOR) { + return cost1 > cost2; + } else { + return cost1 < cost2; + } + } + + private void updateExterior(long affectedNode, double affectedCost, HugeLongPriorityQueue toTrim, BitSet exterior) { + if (!toTrim.containsElement(affectedNode)) { + toTrim.add(affectedNode, affectedCost); //add it to pq + } else { + //it is still in the queue, but it is not a leaf anymore, so it's value is obsolete + toTrim.set(affectedNode, affectedCost); + } + exterior.set(affectedNode); //and mark it in the exterior + } + + private long findNextValidLeaf(HugeLongPriorityQueue toTrim, BitSet exterior) { + while (!exterior.get(toTrim.top())) { //not valid frontier nodes anymore, just ignore + toTrim.pop(); //as we said, pq does not have a direct remove method + } + return toTrim.top(); + } + + private void fillChildren(long newRoot, BitSet rootNodeAdjacent, HugeLongArray parent, BitSet included) { + graph.forEachRelationship(newRoot, (s, t) -> { + //relevant are only those nodes which are currently + //in the k-tree + if (parent.get(t) == s && included.get(t)) { + rootNodeAdjacent.set(t); + } + return true; + }); + } + + private void relaxNode( + long node, + HugeLongPriorityQueue priorityQueue, + HugeLongArray parent, + SpanningTree spanningTree + ) { + graph.forEachRelationship(node, (s, t) -> { + if (parent.get(t) == s) { + //TODO: work's only on mst edges for now (should be doable to re-find an k-MST from whole graph) + if (!priorityQueue.containsElement(t)) { + priorityQueue.add(t, spanningTree.costToParent(t)); + } + + } + return true; + }); + } + } + + diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java index e0e3b7bba99..70c2451b297 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java @@ -52,8 +52,7 @@ public Task progressTask( ) { return Tasks.task( taskName(), - Tasks.leaf("SpanningTree", graph.nodeCount()), - Tasks.leaf("Add relationship weights"), + Tasks.leaf("SpanningTree", graph.relationshipCount()), Tasks.leaf("Remove relationships") ); } diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/influenceMaximization/CelfTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/influenceMaximization/CelfTest.java new file mode 100644 index 00000000000..f35c40dadce --- /dev/null +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/influenceMaximization/CelfTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.impl.influenceMaximization; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.beta.generator.RandomGraphGenerator; +import org.neo4j.gds.beta.generator.RelationshipDistribution; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.influenceMaximization.CELF; + +import static org.assertj.core.api.Assertions.assertThat; + + class CelfTest { + + @ParameterizedTest + @ValueSource(ints = {2, 7}) + void shouldNotReturnNegativeGains(int seedSize) { + var graph = RandomGraphGenerator + .builder() + .averageDegree(5) + .relationshipDistribution(RelationshipDistribution.POWER_LAW) + .direction(Direction.DIRECTED) + .nodeCount(60) + .seed(42) + .build() + .generate(); + + var celf = new CELF(graph, seedSize, 0.1, 3, Pools.DEFAULT, 1, 10, 5, ProgressTracker.NULL_TRACKER).compute(); + for (var a : celf) { + assertThat(a.value).isNotNegative(); + } + } +} diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java index 4b1c2abee89..f0ecfa60b06 100644 --- a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java @@ -19,28 +19,37 @@ */ package org.neo4j.gds.impl.spanningtree; +import org.apache.commons.lang3.mutable.MutableLong; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.neo4j.gds.Orientation; +import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.gdl.GdlFactory; import org.neo4j.gds.spanningtree.Prim; -import org.neo4j.gds.spanningtree.SpanningTree; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; +import java.util.HashSet; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.assertj.Extractors.removingThreadId; +import static org.neo4j.gds.assertj.Extractors.replaceTimings; /** - * 1 - * (x) >(a)---(d) (x) (a) (d) - * /3 \2 /3 => / / - * (b)---(c) (b) (c) - * 1 + * 1 + * (x), (a)---(d) (x) (a) (d) + * /3 \2 /3 => / / + * (b)---(c) (b) (c) + * 1 */ @GdlExtension class KSpanningTreeTest { @@ -66,6 +75,14 @@ class KSpanningTreeTest { @Inject private IdFunction idFunction; + private static final int OFFSET = 5; + + @GdlGraph(idOffset = OFFSET, orientation = Orientation.UNDIRECTED, graphNamePrefix = "offset") + private static final String DB_CYPHER_WITH_OFFSET = DB_CYPHER; + + @Inject + private Graph offsetGraph; + private int a, b, c, d, x; @BeforeEach @@ -79,49 +96,216 @@ void setUp() { @Test void testMaximumKSpanningTree() { - final SpanningTree spanningTree = new KSpanningTree( - graph, - Prim.MAX_OPERATOR, - a, - 2, - ProgressTracker.NULL_TRACKER - ) + var spanningTree = new KSpanningTree(graph, Prim.MAX_OPERATOR, a, 2, ProgressTracker.NULL_TRACKER) .compute(); - assertEquals(spanningTree.head(a), spanningTree.head(b)); - assertEquals(spanningTree.head(c), spanningTree.head(d)); - assertNotEquals(spanningTree.head(a), spanningTree.head(c)); - assertNotEquals(spanningTree.head(a), spanningTree.head(x)); - assertNotEquals(spanningTree.head(c), spanningTree.head(x)); + assertThat(spanningTree).matches(tree -> tree.head(a) == tree.head(b) ^ tree.head(c) == tree.head(d)); + assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(c)); + assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(x)); + assertThat(spanningTree.head(c)).isNotEqualTo(spanningTree.head(x)); } @Test void testMinimumKSpanningTree() { - final SpanningTree spanningTree = new KSpanningTree( + var spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, a, 2, ProgressTracker.NULL_TRACKER) + .compute(); + + assertThat(spanningTree).matches(tree -> tree.head(a) == tree.head(d) ^ tree.head(b) == tree.head(c)); + assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(b)); + assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(x)); + assertThat(spanningTree.head(b)).isNotEqualTo(spanningTree.head(x)); + } + + @Test + void testNeoIdsWithOffset() { + var spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, 0, 2, ProgressTracker.NULL_TRACKER).compute(); + var otherSpanningTree = new KSpanningTree(offsetGraph, Prim.MIN_OPERATOR, OFFSET, 2, ProgressTracker.NULL_TRACKER).compute(); + + assertThat(spanningTree.parentArray().toArray()) + .containsExactly(otherSpanningTree.parentArray().toArray()); + } + + @Test + void shouldProduceSingleConnectedTree() { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (b)-[:TYPE {cost: 1.0}]->(a)" + + ", (c)-[:TYPE {cost: 20.0}]->(b)" + + ", (d)-[:TYPE {cost: 30.0}]->(c)" + + ", (d)-[:TYPE {cost: 1.0}]->(e)" + ); + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("d"); + + var k = 3; + var spanningTree = new KSpanningTree( graph, Prim.MIN_OPERATOR, - a, - 2, + startNode, + k, ProgressTracker.NULL_TRACKER - ) - .compute(); + ).compute(); + + // if there are more than k nodes then there is more than one root + // meaning there is more than one tree (or the tree is broken) + var nodesInTree = new HashSet(); + spanningTree.forEach((s, t, __) -> { + nodesInTree.add(s); + nodesInTree.add(t); + return true; + }); - assertEquals(spanningTree.head(a), spanningTree.head(d)); - assertEquals(spanningTree.head(b), spanningTree.head(c)); - assertNotEquals(spanningTree.head(a), spanningTree.head(b)); - assertNotEquals(spanningTree.head(a), spanningTree.head(x)); - assertNotEquals(spanningTree.head(b), spanningTree.head(x)); + assertThat(nodesInTree.size()).isEqualTo(k); } + @ParameterizedTest + @CsvSource({"2,1.0", "3,2.0"}) + void shouldProduceSingleTreeWithKMinusOneEdges(int k, double expected) { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (f:Node)" + + ", (b)-[:TYPE {cost: 1.0}]->(a)" + + ", (c)-[:TYPE {cost: 20.0}]->(b)" + + ", (d)-[:TYPE {cost: 30.0}]->(c)" + + ", (d)-[:TYPE {cost: 1.0}]->(e)" + + ", (e)-[:TYPE {cost: 1.0}]->(f)" + ); + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("d"); + + + var spanningTree = new KSpanningTree( + graph, + Prim.MIN_OPERATOR, + startNode, + k, + ProgressTracker.NULL_TRACKER + ).compute(); + + var counter = new MutableLong(0); + spanningTree.forEach((__, ___, ____) -> { + counter.add(1); + return true; + }); + + assertThat(counter.getValue()).isEqualTo(k - 1); + + assertThat(spanningTree.totalWeight()).isEqualTo(expected); + } + + @Test - @Disabled("Need to extend GdlGraph to generate offset node IDs and fix the test") - void testNeoIdsWithOffset() { - SpanningTree spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, 0, 2, ProgressTracker.NULL_TRACKER) - .compute(); + void worstCaseForPruningLeaves() { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (f:Node)" + + ", (g:Node)" + + ", (a)-[:TYPE {cost: 9.0}]->(b)" + + ", (b)-[:TYPE {cost: 0.0}]->(c)" + + ", (c)-[:TYPE {cost: 0.0}]->(d)" + + ", (a)-[:TYPE {cost: 1.0}]->(e)" + + ", (e)-[:TYPE {cost: 1.0}]->(f)" + + ", (f)-[:TYPE {cost: 1.0}]->(g)" + + ); + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("a"); - SpanningTree otherSpanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, 5, 2, ProgressTracker.NULL_TRACKER) - .compute(); - assertEquals(spanningTree, otherSpanningTree); + var spanningTree = new KSpanningTree( + graph, + Prim.MIN_OPERATOR, + startNode, + 4, + ProgressTracker.NULL_TRACKER + ).compute(); + + var counter = new MutableLong(0); + spanningTree.forEach((__, ___, ____) -> { + counter.add(1); + return true; + }); + + assertThat(counter.getValue()).isEqualTo(4 - 1); + assertThat(spanningTree.totalWeight()).isEqualTo(3.0); + //here a bad case for pruning just leaves + /// edge weight should be eliminated for the final solution but it is not because + //its leaves are not good. + } + + @Test + void shouldWorkForComponentSmallerThanK() { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (f:Node)" + + ", (g:Node)" + + ", (a)-[:TYPE {cost: 1.0}]->(b)" + + ", (b)-[:TYPE {cost: 1.0}]->(c)" + + ", (c)-[:TYPE {cost: 1.0}]->(d)"); + + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("a"); + + var spanningTree = new KSpanningTree( + graph, + Prim.MIN_OPERATOR, + startNode, + 5, + ProgressTracker.NULL_TRACKER + ).compute(); + + assertThat(spanningTree.effectiveNodeCount()).isEqualTo(4); + + } + + @Test + void shouldLogProgress() { + var config = KSpanningTreeBaseConfigImpl.builder().sourceNode(idFunction.of("a")).k(2).build(); + var factory = new KSpanningTreeAlgorithmFactory<>(); + var log = Neo4jProxy.testLog(); + var progressTracker = new TestProgressTracker( + factory.progressTask(graph, config), + log, + 1, + EmptyTaskRegistryFactory.INSTANCE + ); + factory.build(graph, config, progressTracker).compute(); + assertThat(log.getMessages(TestLog.INFO)) + .extracting(removingThreadId()) + .extracting(replaceTimings()) + .containsExactly( + "KSpanningTree :: Start", + "KSpanningTree :: SpanningTree :: Start", + "KSpanningTree :: SpanningTree 30%", + "KSpanningTree :: SpanningTree 50%", + "KSpanningTree :: SpanningTree 80%", + "KSpanningTree :: SpanningTree 100%", + "KSpanningTree :: SpanningTree :: Finished", + "KSpanningTree :: Remove relationships :: Start", + "KSpanningTree :: Remove relationships 20%", + "KSpanningTree :: Remove relationships 40%", + "KSpanningTree :: Remove relationships 60%", + "KSpanningTree :: Remove relationships 100%", + "KSpanningTree :: Remove relationships :: Finished", + "KSpanningTree :: Finished" + ); + } + } diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java index f572ba0c373..dc8ec4e7680 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java @@ -92,7 +92,7 @@ public ComputationResultConsumer. + */ +package org.neo4j.gds.cypher; + +import org.apache.commons.lang3.mutable.MutableLong; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseNotFoundException; +import org.neo4j.gds.BaseProc; +import org.neo4j.gds.ProcPreconditions; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.core.utils.ProgressTimer; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.procedure.Description; +import org.neo4j.procedure.Name; +import org.neo4j.procedure.Procedure; + +import java.util.stream.Stream; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; +import static org.neo4j.procedure.Mode.WRITE; + +public class DropCypherDbProc extends BaseProc { + + private static final String DESCRIPTION = "Drop a database backed by an in-memory graph"; + + @Procedure(name = "gds.alpha.drop.cypherdb", mode = WRITE) + @Description(DESCRIPTION) + public Stream dropDb( + @Name(value = "dbName") String dbName + ) { + ProcPreconditions.check(); + + DropCypherDbResult result = runWithExceptionLogging( + "Drop in-memory Cypher database failed", + () -> { + GraphCreateCypherDbProc.validateNeo4jEnterpriseEdition(databaseService); + var dbms = GraphDatabaseApiProxy.resolveDependency(databaseService, DatabaseManagementService.class); + validateDatabaseName(dbName, dbms); + var dropMillis = new MutableLong(0); + try (var ignored = ProgressTimer.start(dropMillis::setValue)) { + dbms.dropDatabase(dbName); + } + return new DropCypherDbResult(dbName, dropMillis.getValue()); + } + ); + + return Stream.of(result); + } + + private static void validateDatabaseName(String dbName, DatabaseManagementService dbms) { + if (!dbms.listDatabases().contains(dbName)) { + throw new DatabaseNotFoundException(formatWithLocale("A database with name `%s` does not exist", dbName)); + } + + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(dbName); + if (graphName == null) { + throw new IllegalArgumentException(formatWithLocale( + "Database with name `%s` is not an in-memory database", + dbName + )); + } + } + + public static class DropCypherDbResult { + public final String dbName; + public final long dropMillis; + + public DropCypherDbResult(String dbName, long dropMillis) { + this.dbName = dbName; + this.dropMillis = dropMillis; + } + } +} diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/GraphCreateCypherDbProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/GraphCreateCypherDbProc.java index 6402f4d0607..aff8e66825f 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/GraphCreateCypherDbProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/GraphCreateCypherDbProc.java @@ -27,6 +27,7 @@ import org.neo4j.gds.compat.StorageEngineProxy; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.storageengine.InMemoryDatabaseCreator; +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -52,7 +53,7 @@ public Stream createDb( CreateCypherDbResult result = runWithExceptionLogging( "In-memory Cypher database creation failed", () -> { - validateNeo4jEnterpriseEdition(); + validateNeo4jEnterpriseEdition(databaseService); MutableLong createMillis = new MutableLong(0); try (ProgressTimer ignored = ProgressTimer.start(createMillis::setValue)) { InMemoryDatabaseCreator.createDatabase(databaseService, username(), graphName, dbName); @@ -78,7 +79,7 @@ public CreateCypherDbResult(String dbName, String graphName, long createMillis) } } - private void validateNeo4jEnterpriseEdition() { + static void validateNeo4jEnterpriseEdition(GraphDatabaseService databaseService) { var edition = StorageEngineProxy.dbmsEdition(databaseService); if (!(edition == Edition.ENTERPRISE)) { throw new DatabaseManagementException(formatWithLocale( diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/influenceMaximization/CelfNodeProperties.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/influenceMaximization/CelfNodeProperties.java index 1b39237b442..8e4036196ef 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/influenceMaximization/CelfNodeProperties.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/influenceMaximization/CelfNodeProperties.java @@ -33,7 +33,7 @@ class CelfNodeProperties implements DoubleNodePropertyValues { } @Override - public long size() { + public long nodeCount() { return totalGraphNodeCount; } diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java index 0b6163ca5f9..6b2ebe2bfe3 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java @@ -23,7 +23,9 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.executor.ComputationResult; -public class ScalePropertiesProc { +public final class ScalePropertiesProc { + + private ScalePropertiesProc() {} static NodePropertyValues nodeProperties(ComputationResult computationResult) { var size = computationResult.graph().nodeCount(); @@ -31,7 +33,7 @@ static NodePropertyValues nodeProperties(ComputationResult computationResult, ExecutionContext executionContext ) { - computationResult.graphStore().addRelationshipType( - RelationshipType.of(computationResult.config().mutateRelationshipType()), - computationResult.result() - ); + computationResult.graphStore().addRelationshipType(computationResult.result()); resultBuilder.withRelationshipsWritten(computationResult.result().topology().elementCount()); } diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java deleted file mode 100644 index fa3fa7c61c5..00000000000 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.neo4j.gds.catalog.GraphProjectProc; -import org.neo4j.gds.functions.IsFiniteFunc; -import org.neo4j.gds.shortestpaths.AllShortestPathsProc; -import org.neo4j.graphdb.Result; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class AllShortestPathsDocTest extends BaseProcTest { - private static final String NL = System.lineSeparator(); - public static final String DB_CYPHER = "CREATE " + - " (a:Loc {name:'A'})," + - " (b:Loc {name:'B'})," + - " (c:Loc {name:'C'})," + - " (d:Loc {name:'D'})," + - " (e:Loc {name:'E'})," + - " (f:Loc {name:'F'})," + - " (a)-[:ROAD {cost:50}]->(b)," + - " (a)-[:ROAD {cost:50}]->(c)," + - " (a)-[:ROAD {cost:100}]->(d)," + - " (b)-[:ROAD {cost:40}]->(d)," + - " (c)-[:ROAD {cost:40}]->(d)," + - " (c)-[:ROAD {cost:80}]->(e)," + - " (d)-[:ROAD {cost:30}]->(e)," + - " (d)-[:ROAD {cost:80}]->(f)," + - " (e)-[:ROAD {cost:40}]->(f);"; - - @BeforeEach - void setupGraph() throws Exception { - registerProcedures(AllShortestPathsProc.class, GraphProjectProc.class); - registerFunctions(IsFiniteFunc.class); - runQuery(DB_CYPHER); - } - - @Test - void shouldStream() { - var createQuery = "CALL gds.graph.project(" + - " 'nativeGraph', " + - " 'Loc', " + - " {" + - " ROAD: {" + - " type: 'ROAD'," + - " properties: 'cost'" + - " }" + - " }" + - ")" + - "YIELD graphName"; - runQuery(createQuery); - String query = " CALL gds.alpha.allShortestPaths.stream('nativeGraph', {" + - " relationshipWeightProperty: 'cost'" + - " })" + - " YIELD sourceNodeId, targetNodeId, distance" + - " WITH sourceNodeId, targetNodeId, distance" + - " WHERE gds.util.isFinite(distance) = true" + - - " MATCH (source:Loc) WHERE id(source) = sourceNodeId" + - " MATCH (target:Loc) WHERE id(target) = targetNodeId" + - " WITH source, target, distance WHERE source <> target" + - - " RETURN source.name AS source, target.name AS target, distance" + - " ORDER BY distance DESC, source ASC, target ASC" + - " LIMIT 10"; - - String actual = runQuery(query, Result::resultAsString); - String expected = "+----------------------------+" + NL + - "| source | target | distance |" + NL + - "+----------------------------+" + NL + - "| \"A\" | \"F\" | 160.0 |" + NL + - "| \"A\" | \"E\" | 120.0 |" + NL + - "| \"B\" | \"F\" | 110.0 |" + NL + - "| \"C\" | \"F\" | 110.0 |" + NL + - "| \"A\" | \"D\" | 90.0 |" + NL + - "| \"B\" | \"E\" | 70.0 |" + NL + - "| \"C\" | \"E\" | 70.0 |" + NL + - "| \"D\" | \"F\" | 70.0 |" + NL + - "| \"A\" | \"B\" | 50.0 |" + NL + - "| \"A\" | \"C\" | 50.0 |" + NL + - "+----------------------------+" + NL + - "10 rows" + NL; - - assertEquals(expected, actual); - } - - @Test - void shouldStreamWithCypherProjection() { - var createQuery = " CALL gds.graph.project.cypher(" + - " 'cypherGraph'," + - " 'MATCH (n:Loc) RETURN id(n) AS id', " + - " 'MATCH (n:Loc)-[r:ROAD]-(p:Loc) RETURN id(n) AS source, id(p) AS target, r.cost AS cost'" + - " ) " + - " YIELD graphName"; - runQuery(createQuery); - String query = " CALL gds.alpha.allShortestPaths.stream('cypherGraph', {" + - " relationshipWeightProperty: 'cost'" + - " })" + - " YIELD sourceNodeId, targetNodeId, distance" + - " WITH sourceNodeId, targetNodeId, distance" + - " WHERE gds.util.isFinite(distance) = true" + - - " MATCH (source:Loc) WHERE id(source) = sourceNodeId" + - " MATCH (target:Loc) WHERE id(target) = targetNodeId" + - " WITH source, target, distance WHERE source <> target" + - - " RETURN source.name AS source, target.name AS target, distance" + - " ORDER BY distance DESC, source ASC, target ASC" + - " LIMIT 10"; - - String actual = runQuery(query, Result::resultAsString); - String expected = "+----------------------------+" + NL + - "| source | target | distance |" + NL + - "+----------------------------+" + NL + - "| \"A\" | \"F\" | 160.0 |" + NL + - "| \"F\" | \"A\" | 160.0 |" + NL + - "| \"A\" | \"E\" | 120.0 |" + NL + - "| \"E\" | \"A\" | 120.0 |" + NL + - "| \"B\" | \"F\" | 110.0 |" + NL + - "| \"C\" | \"F\" | 110.0 |" + NL + - "| \"F\" | \"B\" | 110.0 |" + NL + - "| \"F\" | \"C\" | 110.0 |" + NL + - "| \"A\" | \"D\" | 90.0 |" + NL + - "| \"D\" | \"A\" | 90.0 |" + NL + - "+----------------------------+" + NL + - "10 rows" + NL; - - assertEquals(expected, actual); - } -} diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/modularity/ModularityStatsProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/modularity/ModularityStatsProcTest.java index 618085fcf93..5fac0449ecb 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/modularity/ModularityStatsProcTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/modularity/ModularityStatsProcTest.java @@ -38,10 +38,10 @@ class ModularityStatsProcTest extends BaseProcTest { @Neo4jGraph static final String GRAPH = "CREATE " + - " (a1: Node { communityId: 0 })," + - " (a2: Node { communityId: 0 })," + + " (a1: Node { communityId: 10 })," + + " (a2: Node { communityId: 10 })," + " (a3: Node { communityId: 5 })," + - " (a4: Node { communityId: 0 })," + + " (a4: Node { communityId: 10 })," + " (a5: Node { communityId: 5 })," + " (a6: Node { communityId: 5 })," + diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java index d343b89dab0..e2ad893d7ee 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java @@ -30,17 +30,16 @@ import org.neo4j.gds.extension.Neo4jGraph; import java.util.HashMap; +import java.util.HashSet; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; class KSpanningTreeWriteProcTest extends BaseProcTest { - private static String GRAPH_NAME = "graph"; + private static final String GRAPH_NAME = "graph"; @Neo4jGraph - static final String DB_CYPHER = + private static final String DB_CYPHER = "CREATE (a:Node {name:'a'})\n" + "CREATE (b:Node {name:'b'})\n" + "CREATE (c:Node {name:'c'})\n" + @@ -54,7 +53,7 @@ class KSpanningTreeWriteProcTest extends BaseProcTest { " (d)-[:TYPE {w:3.0}]->(c)"; @Inject - IdFunction idFunction; + private IdFunction idFunction; @BeforeEach void setupGraph() throws Exception { @@ -80,22 +79,25 @@ void testMax() { .yields("preProcessingMillis", "computeMillis", "writeMillis"); runQueryWithRowConsumer(query, row -> { - assertTrue(row.getNumber("preProcessingMillis").longValue() >= 0); - assertTrue(row.getNumber("writeMillis").longValue() >= 0); - assertTrue(row.getNumber("computeMillis").longValue() >= 0); + assertThat(row.getNumber("preProcessingMillis").longValue() >= 0).isTrue(); + assertThat(row.getNumber("writeMillis").longValue() >= 0).isTrue(); + assertThat(row.getNumber("computeMillis").longValue() >= 0).isTrue(); }); - final HashMap communities = new HashMap<>(); + final HashMap communities = new HashMap<>(); + final HashSet distinctCommunities = new HashSet<>(); runQueryWithRowConsumer("MATCH (n) WHERE n.partition IS NOT NULL RETURN n.name as name, n.partition as p", row -> { final String name = row.getString("name"); - final long p = row.getNumber("p").longValue(); + final int p = row.getNumber("p").intValue(); communities.put(name, p); + distinctCommunities.add(p); + }); - assertEquals(communities.get("a"), communities.get("b")); - assertEquals(communities.get("d"), communities.get("c")); - assertNotEquals(communities.get("a"), communities.get("c")); + assertThat(communities).matches(c -> c.get("a").equals(c.get("b")) ^ c.get("c").equals(c.get("d"))); + assertThat(communities.get("a")).isNotEqualTo(communities.get("c")); + assertThat(distinctCommunities.size()).isEqualTo(3); } @Test @@ -109,24 +111,22 @@ void testMin() { .addParameter("writeProperty", "partition") .yields("preProcessingMillis", "computeMillis", "writeMillis"); - runQueryWithRowConsumer(query, row -> { - assertTrue(row.getNumber("preProcessingMillis").longValue() >= 0); - assertTrue(row.getNumber("writeMillis").longValue() >= 0); - assertTrue(row.getNumber("computeMillis").longValue() >= 0); + assertThat(row.getNumber("preProcessingMillis").longValue() >= 0).isTrue(); + assertThat(row.getNumber("writeMillis").longValue() >= 0).isTrue(); + assertThat(row.getNumber("computeMillis").longValue() >= 0).isTrue(); }); final HashMap communities = new HashMap<>(); - + final HashSet distinctCommunities = new HashSet<>(); runQueryWithRowConsumer("MATCH (n) WHERE n.partition IS NOT NULL RETURN n.name as name, n.partition as p", row -> { final String name = row.getString("name"); final int p = row.getNumber("p").intValue(); communities.put(name, p); + distinctCommunities.add(p); }); - assertEquals(communities.get("a"), communities.get("d")); - assertEquals(communities.get("b"), communities.get("c")); - assertNotEquals(communities.get("a"), communities.get("b")); + assertThat(communities).matches(c -> c.get("a").equals(c.get("d")) ^ c.get("b").equals(c.get("c"))); + assertThat(distinctCommunities.size()).isEqualTo(3); } - } diff --git a/annotations/src/main/java/org/neo4j/gds/annotation/DataClass.java b/annotations/src/main/java/org/neo4j/gds/annotation/DataClass.java deleted file mode 100644 index 513dbe9689b..00000000000 --- a/annotations/src/main/java/org/neo4j/gds/annotation/DataClass.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.annotation; - -import org.immutables.value.Value; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target({ElementType.PACKAGE, ElementType.TYPE}) -@Retention(RetentionPolicy.CLASS) -@Value.Style( - allParameters = true, - builderVisibility = Value.Style.BuilderVisibility.SAME, - clearBuilder = true, - deepImmutablesDetection = true, - deferCollectionAllocation = true, - depluralize = true, - headerComments = true, - jdkOnly = true, - optionalAcceptNullable = true, - typeAbstract = "Abstract*", - typeImmutable = "*", - visibility = Value.Style.ImplementationVisibility.SAME -) -public @interface DataClass { -} diff --git a/annotations/src/main/resources/META-INF/annotations/org.immutables.value.immutable b/annotations/src/main/resources/META-INF/annotations/org.immutables.value.immutable index b067608b47a..cf5adfc05a9 100644 --- a/annotations/src/main/resources/META-INF/annotations/org.immutables.value.immutable +++ b/annotations/src/main/resources/META-INF/annotations/org.immutables.value.immutable @@ -1,2 +1 @@ -org.neo4j.gds.annotation.DataClass org.neo4j.gds.annotation.ValueClass diff --git a/build.gradle b/build.gradle index c767e083418..165c9eec480 100644 --- a/build.gradle +++ b/build.gradle @@ -31,11 +31,23 @@ ext { project(':neo4j-kernel-adapter-4.4'), project(':neo4j-kernel-adapter-5.1'), project(':neo4j-kernel-adapter-5.2'), + project(':neo4j-kernel-adapter-5.3'), + project(':neo4j-kernel-adapter-5.4'), + project(':neo4j-kernel-adapter-5.5'), + project(':neo4j-kernel-adapter-5.6'), + project(':neo4j-kernel-adapter-5.7'), + project(':neo4j-kernel-adapter-5.8'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), project(':storage-engine-adapter-5.1'), project(':storage-engine-adapter-5.2'), + project(':storage-engine-adapter-5.3'), + project(':storage-engine-adapter-5.4'), + project(':storage-engine-adapter-5.5'), + project(':storage-engine-adapter-5.6'), + project(':storage-engine-adapter-5.7'), + project(':storage-engine-adapter-5.8'), ] ] } diff --git a/collections/src/main/java/org/neo4j/gds/mem/HugeArrays.java b/collections/src/main/java/org/neo4j/gds/mem/HugeArrays.java index a792c21684e..8f2bc3f6706 100644 --- a/collections/src/main/java/org/neo4j/gds/mem/HugeArrays.java +++ b/collections/src/main/java/org/neo4j/gds/mem/HugeArrays.java @@ -51,7 +51,7 @@ public static int exclusiveIndexOfPage(long index) { } public static long indexFromPageIndexAndIndexInPage(int pageIndex, int indexInPage) { - return (pageIndex << PAGE_SHIFT) | indexInPage; + return ((long) pageIndex << PAGE_SHIFT) | indexInPage; } public static int numberOfPages(long capacity) { diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index 8ef9e8ffcbb..64bf9b8c087 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -28,6 +28,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -40,6 +41,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; import org.neo4j.gds.compat.PropertyReference; @@ -108,7 +110,10 @@ import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; @@ -124,6 +129,7 @@ import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.ValueGroup; +import org.neo4j.values.virtual.MapValue; import java.io.IOException; import java.nio.file.Path; @@ -183,6 +189,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, @@ -414,7 +425,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL: return new InputEntityIdVisitor.Long() { @@ -434,20 +445,22 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } }; case INTEGER: + var globalGroup = groups.get(Group.GLOBAL.id()); + return new InputEntityIdVisitor.Long() { @Override public void visitNodeId(InputEntityVisitor visitor, long id) { - visitor.id(id, Group.GLOBAL); + visitor.id(id, globalGroup); } @Override public void visitSourceId(InputEntityVisitor visitor, long id) { - visitor.startId(id, Group.GLOBAL); + visitor.startId(id, globalGroup); } @Override public void visitTargetId(InputEntityVisitor visitor, long id) { - visitor.endId(id, Group.GLOBAL); + visitor.endId(id, globalGroup); } }; default: @@ -456,21 +469,22 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(Group.GLOBAL.id()); return new InputEntityIdVisitor.String() { @Override public void visitNodeId(InputEntityVisitor visitor, String id) { - visitor.id(id, Group.GLOBAL); + visitor.id(id, globalGroup); } @Override public void visitSourceId(InputEntityVisitor visitor, String id) { - visitor.startId(id, Group.GLOBAL); + visitor.startId(id, globalGroup); } @Override public void visitTargetId(InputEntityVisitor visitor, String id) { - visitor.endId(id, Group.GLOBAL); + visitor.endId(id, globalGroup); } }; } @@ -785,5 +799,19 @@ public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, Curso idGenerator.nextConsecutiveIdRange(size, false, cursorContext); } + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService).name()); + } } diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java index 88226c3dbf0..ac658882bf2 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java @@ -77,4 +77,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab } config.set(GraphDatabaseSettings.mode, mode); } + + @Override + public String secondaryModeName() { + return "Read Replica"; + } } diff --git a/cypher/4.4/storage-engine-adapter/build.gradle b/compatibility/4.4/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/4.4/storage-engine-adapter/build.gradle rename to compatibility/4.4/storage-engine-adapter/build.gradle diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java similarity index 97% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java index 7ac94920456..ef689ef60ab 100644 --- a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java +++ b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java @@ -22,6 +22,8 @@ import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.configuration.Config; import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.id.IdController; import org.neo4j.internal.id.IdGeneratorFactory; @@ -73,6 +75,10 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-44"; + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_4_4, StorageEngineFactory.class); + } + private final InMemoryMetaDataProviderImpl metadataProvider = new InMemoryMetaDataProviderImpl(); @Override diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java similarity index 55% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java index a07942bdb6d..98905edd331 100644 --- a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java +++ b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java @@ -17,42 +17,29 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.internal.recordstorage; +package org.neo4j.gds.compat._44; +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; import org.neo4j.storageengine.api.TransactionId; -import org.neo4j.storageengine.api.TransactionIdStore; -import org.neo4j.util.concurrent.ArrayQueueOutOfOrderSequence; -import org.neo4j.util.concurrent.OutOfOrderSequence; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { -public abstract class AbstractTransactionIdStore implements TransactionIdStore { - private final AtomicLong committingTransactionId; - protected final OutOfOrderSequence closedTransactionId; - private final AtomicReference committedTransactionId; - private final long previouslyCommittedTxId; - private final int initialTransactionChecksum; - private final long previouslyCommittedTxCommitTimestamp; - - public AbstractTransactionIdStore() { - this(1L, -559063315, 0L, 0L, 64L); + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata(metaData[0], new LogPosition(metaData[1], metaData[2])); } - public AbstractTransactionIdStore( + @Override + protected void initLastCommittedAndClosedTransactionId( long previouslyCommittedTxId, int checksum, long previouslyCommittedTxCommitTimestamp, - long previouslyCommittedTxLogVersion, - long previouslyCommittedTxLogByteOffset + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion ) { - this.committingTransactionId = new AtomicLong(); - this.closedTransactionId = new ArrayQueueOutOfOrderSequence(-1L, 100, new long[1]); - this.committedTransactionId = new AtomicReference<>(new TransactionId(1L, -559063315, 0L)); - - assert previouslyCommittedTxId >= 1L : "cannot start from a tx id less than BASE_TX_ID"; - this.setLastCommittedAndClosedTransactionId( previouslyCommittedTxId, checksum, @@ -61,21 +48,9 @@ public AbstractTransactionIdStore( previouslyCommittedTxLogVersion, getEmptyCursorContext() ); - this.previouslyCommittedTxId = previouslyCommittedTxId; - this.initialTransactionChecksum = checksum; - this.previouslyCommittedTxCommitTimestamp = previouslyCommittedTxCommitTimestamp; - } - - protected abstract CursorContext getEmptyCursorContext(); - - public long nextCommittingTransactionId() { - return this.committingTransactionId.incrementAndGet(); - } - - public long committingTransactionId() { - return this.committingTransactionId.get(); } + @Override public synchronized void transactionCommitted( long transactionId, int checksum, @@ -84,31 +59,11 @@ public synchronized void transactionCommitted( ) { TransactionId current = this.committedTransactionId.get(); if (current == null || transactionId > current.transactionId()) { - this.committedTransactionId.set(new TransactionId(transactionId, checksum, commitTimestamp)); + this.committedTransactionId.set(transactionId(transactionId, checksum, commitTimestamp)); } - - } - - public long getLastCommittedTransactionId() { - return this.committedTransactionId.get().transactionId(); - } - - public TransactionId getLastCommittedTransaction() { - return this.committedTransactionId.get(); - } - - public TransactionId getUpgradeTransaction() { - return new TransactionId( - this.previouslyCommittedTxId, - this.initialTransactionChecksum, - this.previouslyCommittedTxCommitTimestamp - ); - } - - public long getLastClosedTransactionId() { - return this.closedTransactionId.getHighestGapFreeNumber(); } + @Override public void setLastCommittedAndClosedTransactionId( long transactionId, int checksum, @@ -118,14 +73,11 @@ public void setLastCommittedAndClosedTransactionId( CursorContext cursorContext ) { this.committingTransactionId.set(transactionId); - this.committedTransactionId.set(new TransactionId(transactionId, checksum, commitTimestamp)); + this.committedTransactionId.set(transactionId(transactionId, checksum, commitTimestamp)); this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); } - public void transactionClosed(long transactionId, long logVersion, long byteOffset, CursorContext cursorContext) { - this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset}); - } - + @Override public void resetLastClosedTransaction( long transactionId, long byteOffset, @@ -136,6 +88,30 @@ public void resetLastClosedTransaction( this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset}); } + @Override + public TransactionId getUpgradeTransaction() { + return transactionId( + this.previouslyCommittedTxId, + this.initialTransactionChecksum, + this.previouslyCommittedTxCommitTimestamp + ); + } + + @Override + public void transactionClosed(long transactionId, long logVersion, long byteOffset, CursorContext cursorContext) { + this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset}); + } + + @Override public void flush(CursorContext cursorContext) { } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); + } + + private CursorContext getEmptyCursorContext() { + return CursorContext.NULL; + } } diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index 5d085755c17..17bfd3e1bf0 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -62,7 +63,6 @@ import org.neo4j.internal.batchimport.InputIterable; import org.neo4j.internal.batchimport.Monitor; import org.neo4j.internal.batchimport.input.Collector; -import org.neo4j.internal.batchimport.input.Group; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; import org.neo4j.internal.batchimport.input.InputEntityVisitor; @@ -116,7 +116,10 @@ import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -134,6 +137,7 @@ import org.neo4j.util.Bits; import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; import java.io.IOException; import java.nio.file.Path; @@ -198,6 +202,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, @@ -509,7 +518,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL -> { return new InputEntityIdVisitor.Long() { @@ -530,7 +539,7 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { }; } case INTEGER -> { - var globalGroup = new Group(0, null, null); + var globalGroup = groups.get(null); return new InputEntityIdVisitor.Long() { @Override @@ -554,8 +563,8 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { - var globalGroup = new Group(0, null, null); + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); return new InputEntityIdVisitor.String() { @Override @@ -908,5 +917,19 @@ public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, Curso idGenerator.nextConsecutiveIdRange(size, false, cursorContext); } + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java index 97dbcff99c8..1412919356e 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java @@ -79,4 +79,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab throw new RuntimeException("Could not get the permissions to set the mode field.", e); } } + + @Override + public String secondaryModeName() { + return "Secondary"; + } } diff --git a/cypher/5.1/storage-engine-adapter/build.gradle b/compatibility/5.1/storage-engine-adapter/build.gradle similarity index 96% rename from cypher/5.1/storage-engine-adapter/build.gradle rename to compatibility/5.1/storage-engine-adapter/build.gradle index ac0d4da3b60..d9dcbe96a38 100644 --- a/cypher/5.1/storage-engine-adapter/build.gradle +++ b/compatibility/5.1/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..c416233af83 --- /dev/null +++ b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._51; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported51"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + } +} diff --git a/cypher/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java similarity index 98% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java index bdbf2a266d4..14dd53ef3fe 100644 --- a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java @@ -27,6 +27,8 @@ import org.neo4j.consistency.report.ConsistencySummaryStatistics; import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.batchimport.AdditionalInitialIds; import org.neo4j.internal.batchimport.BatchImporter; @@ -113,6 +115,10 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-51"; + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_1, StorageEngineFactory.class); + } + private final InMemoryMetaDataProviderImpl metadataProvider = new InMemoryMetaDataProviderImpl(); @Override diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java similarity index 74% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java index 0ff4ec50d7c..f370df22453 100644 --- a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java @@ -20,9 +20,9 @@ package org.neo4j.gds.compat._51; import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { @@ -36,6 +36,23 @@ public ClosedTransactionMetadata getLastClosedTransaction() { ); } + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + @Override public void transactionClosed( long transactionId, @@ -69,7 +86,7 @@ public void setLastCommittedAndClosedTransactionId( } @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL_CONTEXT; + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); } } diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index 2d151a763bd..18178a08351 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -62,7 +63,6 @@ import org.neo4j.internal.batchimport.InputIterable; import org.neo4j.internal.batchimport.Monitor; import org.neo4j.internal.batchimport.input.Collector; -import org.neo4j.internal.batchimport.input.Group; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; import org.neo4j.internal.batchimport.input.InputEntityVisitor; @@ -116,7 +116,10 @@ import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -134,6 +137,7 @@ import org.neo4j.util.Bits; import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; import java.io.IOException; import java.nio.file.Path; @@ -198,6 +202,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, @@ -507,7 +516,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL -> { return new InputEntityIdVisitor.Long() { @@ -528,7 +537,7 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { }; } case INTEGER -> { - var globalGroup = new Group(0, null, null); + var globalGroup = groups.get(null); return new InputEntityIdVisitor.Long() { @Override @@ -552,8 +561,8 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { - var globalGroup = new Group(0, null, null); + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); return new InputEntityIdVisitor.String() { @Override @@ -906,5 +915,19 @@ public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, Curso idGenerator.nextConsecutiveIdRange(size, false, cursorContext); } + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java index ef92ef88db5..7dd4139a1e1 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java @@ -79,4 +79,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab throw new RuntimeException("Could not get the permissions to set the mode field.", e); } } + + @Override + public String secondaryModeName() { + return "Secondary"; + } } diff --git a/cypher/5.2/storage-engine-adapter/build.gradle b/compatibility/5.2/storage-engine-adapter/build.gradle similarity index 96% rename from cypher/5.2/storage-engine-adapter/build.gradle rename to compatibility/5.2/storage-engine-adapter/build.gradle index 8af17934fc0..0bc2a320eba 100644 --- a/cypher/5.2/storage-engine-adapter/build.gradle +++ b/compatibility/5.2/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..499d70a28a4 --- /dev/null +++ b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._52; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported52"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + } +} diff --git a/cypher/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java similarity index 98% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java index 4c3fa12ad27..04a977b6d07 100644 --- a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java @@ -27,6 +27,8 @@ import org.neo4j.consistency.report.ConsistencySummaryStatistics; import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.batchimport.AdditionalInitialIds; import org.neo4j.internal.batchimport.BatchImporter; @@ -113,6 +115,10 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-52"; + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_2, StorageEngineFactory.class); + } + private final InMemoryMetaDataProviderImpl metadataProvider = new InMemoryMetaDataProviderImpl(); @Override diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java similarity index 74% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java index 4932d715975..ea4880044c3 100644 --- a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java @@ -20,12 +20,30 @@ package org.neo4j.gds.compat._52; import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override public ClosedTransactionMetadata getLastClosedTransaction() { long[] metaData = this.closedTransactionId.get(); return new ClosedTransactionMetadata( @@ -69,7 +87,7 @@ public void setLastCommittedAndClosedTransactionId( } @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL_CONTEXT; + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); } } diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index 49579510ce0..22542193d48 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -62,7 +63,6 @@ import org.neo4j.internal.batchimport.InputIterable; import org.neo4j.internal.batchimport.Monitor; import org.neo4j.internal.batchimport.input.Collector; -import org.neo4j.internal.batchimport.input.Group; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; import org.neo4j.internal.batchimport.input.InputEntityVisitor; @@ -116,7 +116,10 @@ import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -134,6 +137,7 @@ import org.neo4j.util.Bits; import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; import java.io.IOException; import java.nio.file.Path; @@ -198,6 +202,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, @@ -507,7 +516,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL -> { return new InputEntityIdVisitor.Long() { @@ -528,7 +537,7 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { }; } case INTEGER -> { - var globalGroup = new Group(0, null, null); + var globalGroup = groups.get(null); return new InputEntityIdVisitor.Long() { @Override @@ -552,8 +561,8 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { - var globalGroup = new Group(0, null, null); + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); return new InputEntityIdVisitor.String() { @Override @@ -907,4 +916,19 @@ public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, Curso idGenerator.nextConsecutiveIdRange(size, false, cursorContext); } + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java index abdaa7629e9..d1c06a9af61 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java @@ -79,4 +79,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab throw new RuntimeException("Could not get the permissions to set the mode field.", e); } } + + @Override + public String secondaryModeName() { + return "Secondary"; + } } diff --git a/cypher/5.3/storage-engine-adapter/build.gradle b/compatibility/5.3/storage-engine-adapter/build.gradle similarity index 96% rename from cypher/5.3/storage-engine-adapter/build.gradle rename to compatibility/5.3/storage-engine-adapter/build.gradle index 805b935c705..6db56beaaba 100644 --- a/cypher/5.3/storage-engine-adapter/build.gradle +++ b/compatibility/5.3/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..eada03549ce --- /dev/null +++ b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._53; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported53"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + } +} diff --git a/cypher/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java similarity index 98% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java index 8a60b24501b..cc688fa63ff 100644 --- a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.compat._53; -import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository53; import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.KernelVersion; import org.neo4j.storageengine.api.ClosedTransactionMetadata; @@ -36,11 +36,11 @@ public class InMemoryMetaDataProviderImpl implements MetadataProvider { private final ExternalStoreId externalStoreId; - private final InMemoryLogVersionRepository logVersionRepository; + private final InMemoryLogVersionRepository53 logVersionRepository; private final InMemoryTransactionIdStoreImpl transactionIdStore; InMemoryMetaDataProviderImpl() { - this.logVersionRepository = new InMemoryLogVersionRepository(); + this.logVersionRepository = new InMemoryLogVersionRepository53(); this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); } diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java similarity index 98% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java index 696ee516c8c..6d57e38525f 100644 --- a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -28,6 +28,8 @@ import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; import org.neo4j.function.ThrowingSupplier; import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.batchimport.AdditionalInitialIds; import org.neo4j.internal.batchimport.BatchImporter; @@ -41,8 +43,8 @@ import org.neo4j.internal.batchimport.input.LenientStoreInput; import org.neo4j.internal.id.IdGeneratorFactory; import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; -import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; -import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository53; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory53; import org.neo4j.internal.recordstorage.StoreTokens; import org.neo4j.internal.schema.IndexConfigCompleter; import org.neo4j.internal.schema.SchemaRule; @@ -112,6 +114,10 @@ @ServiceProvider public class InMemoryStorageEngineFactory implements StorageEngineFactory { + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_3, StorageEngineFactory.class); + } + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-53"; @Override @@ -429,7 +435,7 @@ private static List unique( List tokens ) @Override public CommandReaderFactory commandReaderFactory() { - return InMemoryStorageCommandReaderFactory.INSTANCE; + return InMemoryStorageCommandReaderFactory53.INSTANCE; } @Override @@ -472,7 +478,7 @@ public TransactionIdStore readOnlyTransactionIdStore(LogTailMetadata logTailMeta @Override public LogVersionRepository readOnlyLogVersionRepository(LogTailMetadata logTailMetadata) { - return new InMemoryLogVersionRepository(); + return new InMemoryLogVersionRepository53(); } @Override diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java similarity index 74% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java index 73749482bab..845f307c05c 100644 --- a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java @@ -20,12 +20,30 @@ package org.neo4j.gds.compat._53; import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override public ClosedTransactionMetadata getLastClosedTransaction() { long[] metaData = this.closedTransactionId.get(); return new ClosedTransactionMetadata( @@ -69,7 +87,7 @@ public void setLastCommittedAndClosedTransactionId( } @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL_CONTEXT; + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); } } diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository53.java similarity index 88% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository53.java index b491fbc7f32..7a711723491 100644 --- a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository53.java @@ -23,16 +23,16 @@ import java.util.concurrent.atomic.AtomicLong; -public class InMemoryLogVersionRepository implements LogVersionRepository { +public class InMemoryLogVersionRepository53 implements LogVersionRepository { private final AtomicLong logVersion; private final AtomicLong checkpointLogVersion; - public InMemoryLogVersionRepository() { - this(0,0); + public InMemoryLogVersionRepository53() { + this(0, 0); } - private InMemoryLogVersionRepository(long initialLogVersion, long initialCheckpointLogVersion) { + private InMemoryLogVersionRepository53(long initialLogVersion, long initialCheckpointLogVersion) { this.logVersion = new AtomicLong(); this.checkpointLogVersion = new AtomicLong(); this.logVersion.set(initialLogVersion); diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory53.java similarity index 92% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory53.java index 620a8aac11c..2804a31e179 100644 --- a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory53.java @@ -23,9 +23,9 @@ import org.neo4j.storageengine.api.CommandReader; import org.neo4j.storageengine.api.CommandReaderFactory; -public class InMemoryStorageCommandReaderFactory implements CommandReaderFactory { +public class InMemoryStorageCommandReaderFactory53 implements CommandReaderFactory { - public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory(); + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory53(); @Override public CommandReader get(KernelVersion kernelVersion) { diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java diff --git a/compatibility/5.4/neo4j-kernel-adapter/build.gradle b/compatibility/5.4/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..f73ed67ab7f --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.4' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.4' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.4' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.4' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.4' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.4' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.4' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java similarity index 53% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java rename to compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java index 7f7818f3db6..1e38d77de3c 100644 --- a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java @@ -17,22 +17,28 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.gds.compat._44; +package org.neo4j.gds.compat._54; -import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; -import org.neo4j.kernel.impl.transaction.log.LogPosition; -import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; -public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { - public ClosedTransactionMetadata getLastClosedTransaction() { - long[] metaData = this.closedTransactionId.get(); - return new ClosedTransactionMetadata(metaData[0], new LogPosition(metaData[1], metaData[2])); + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.4 compatibility requires JDK17"); } @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL; + public String description() { + return "Neo4j 5.4 (placeholder)"; } } diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..ff39f69c757 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.4 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.4 (placeholder)"; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/BoltTransactionRunnerImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..aa4c808530f --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/BoltTransactionRunnerImpl.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.bookmark.Bookmark; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.transaction.result.AdaptingBoltQuerySubscriber; +import org.neo4j.bolt.protocol.v41.message.request.RoutingContext; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new AdaptingBoltQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSucceeded(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.queryStatistics(); + } + + @Override + public AdaptingBoltQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + AdaptingBoltQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()) + ); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableProcedureImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableProcedureImpl.java new file mode 100644 index 00000000000..c73a48027a0 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableUserAggregationFunctionImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..aa1cabf9039 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatAccessModeImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatAccessModeImpl.java new file mode 100644 index 00000000000..06dfd97bd2d --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatGraphDatabaseAPIImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..06cedb1d578 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatIndexQueryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..adcd4483a1d --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/DatabaseTopologyHelper.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatUsernameAuthSubjectImpl.java similarity index 56% rename from cypher-aggregation/src/main/java/org/neo4j/gds/projection/DatabaseTopologyHelper.java rename to compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatUsernameAuthSubjectImpl.java index 9583ad90f09..974b2cea35d 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/DatabaseTopologyHelper.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatUsernameAuthSubjectImpl.java @@ -17,18 +17,19 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.gds.projection; +package org.neo4j.gds.compat._54; -import org.neo4j.fabric.FabricDatabaseManager; -import org.neo4j.gds.compat.GraphDatabaseApiProxy; -import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; -public final class DatabaseTopologyHelper { +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { - private DatabaseTopologyHelper() {} + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } - public static boolean isCompositeDatabase(GraphDatabaseService graphDatabaseService) { - var databaseManager = GraphDatabaseApiProxy.resolveDependency(graphDatabaseService, FabricDatabaseManager.class); - return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(graphDatabaseService).name()); + @Override + public String executingUser() { + return username; } } diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompositeNodeCursorImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..7fcaba83573 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseLayoutImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..9f655fe2412 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..fb39c02a57e --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..3fe03256e7b --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_4; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.4"; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java new file mode 100644 index 00000000000..95a20fe74d7 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -0,0 +1,934 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public long pageCacheMemory() { + return pageCacheMemory.orElseGet(Configuration.super::pageCacheMemory); + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/NodeLabelIndexLookupImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..bedfd628c17 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/PartitionedStoreScan.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/PartitionedStoreScan.java new file mode 100644 index 00000000000..6d80192595d --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ReferencePropertyReference.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ReferencePropertyReference.java new file mode 100644 index 00000000000..1ccf92014fa --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ScanBasedStoreScanImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..2e84d278532 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..55d9d22773a --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_4; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.4"; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java new file mode 100644 index 00000000000..c854e5a5a80 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/TestLogImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/TestLogImpl.java new file mode 100644 index 00000000000..6e75890ee50 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/TestLogImpl.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} + + diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/VirtualRelationshipImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..3d0f8a96dbb --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/build.gradle b/compatibility/5.4/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..151d3362ee9 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.4' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.4' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.4' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.4' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.4' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..23fb737630d --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported54"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..e997b83e3da --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.4"; + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..d6825dc4902 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.LATEST; + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..f3f3546f7c3 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..7fd6339cb89 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository54; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository54 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository54(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + byteOffset, + logVersion + ); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } + + TransactionIdStore transactionIdStore() { + return this.transactionIdStore; + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java new file mode 100644 index 00000000000..5cc9cc6e713 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..f956b5a6d53 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..6420fa80c4c --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..6cc18b723c0 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..0d3efb743c1 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..9473f7cbb49 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..9abb9283d79 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..a4986e7f275 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -0,0 +1,569 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository54; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory54; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-54"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_4, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + + public static final byte ID = 42; + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + cursorContextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker databaseReadOnlyChecker, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory54.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore(LogTailMetadata logTailMetadata) { + return new InMemoryMetaDataProviderImpl().transactionIdStore(); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository(LogTailMetadata logTailMetadata) { + return new InMemoryLogVersionRepository54(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier throwingSupplier, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..159449bbf2b --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java @@ -0,0 +1,287 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader54; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private final StorageEngineIndexingBehaviour indexingBehaviour = () -> false; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return indexingBehaviour; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader54(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..b0f496aeda2 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java new file mode 100644 index 00000000000..b5663131022 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..3a0ee477755 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4] + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java new file mode 100644 index 00000000000..45feb84ad9a --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._54.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..7f07bf1e11a --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_4; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.4"; + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..f99cb355377 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader54; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader54(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository54.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository54.java new file mode 100644 index 00000000000..c901f56269d --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository54.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository54 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository54() { + this(0, 0); + } + + private InMemoryLogVersionRepository54(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory54.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory54.java new file mode 100644 index 00000000000..4d84f1e0298 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory54.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory54 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory54(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java new file mode 100644 index 00000000000..3b2d28616fd --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java @@ -0,0 +1,325 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._54.InMemoryNodeCursor; +import org.neo4j.gds.compat._54.InMemoryPropertyCursor; +import org.neo4j.gds.compat._54.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._54.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader54 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader54( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/build.gradle b/compatibility/5.5/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..6eed2158956 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.5' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.5' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.5' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.5' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.5' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.5' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.5' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..931a383b46e --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.5 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.5 (placeholder)"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..78e1b894771 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.5 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.5 (placeholder)"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/BoltTransactionRunnerImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..b96e77e2fe3 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/BoltTransactionRunnerImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.bookmark.Bookmark; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.transaction.result.AdaptingBoltQuerySubscriber; +import org.neo4j.bolt.protocol.v41.message.request.RoutingContext; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new AdaptingBoltQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSucceeded(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.queryStatistics(); + } + + @Override + public AdaptingBoltQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + AdaptingBoltQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableProcedureImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableProcedureImpl.java new file mode 100644 index 00000000000..72c0b6545ed --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableUserAggregationFunctionImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..4156ab4cc4e --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatAccessModeImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatAccessModeImpl.java new file mode 100644 index 00000000000..ce96cc3a72a --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatGraphDatabaseAPIImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..0af8537b5a8 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatIndexQueryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..8cad95ee9b5 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatUsernameAuthSubjectImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..148fa614366 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompositeNodeCursorImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..8ee3052f38e --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseLayoutImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..c1dc37d80ef --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..13fdae4cce6 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..7f13c7ebe56 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_5; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.5"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java new file mode 100644 index 00000000000..099e4b3e576 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -0,0 +1,935 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public long pageCacheMemory() { + return pageCacheMemory.orElseGet(Configuration.super::pageCacheMemory); + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/NodeLabelIndexLookupImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..d4b58ae0e40 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/PartitionedStoreScan.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/PartitionedStoreScan.java new file mode 100644 index 00000000000..98463b3150b --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ReferencePropertyReference.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ReferencePropertyReference.java new file mode 100644 index 00000000000..afaaa02f22e --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ScanBasedStoreScanImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..28784106272 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..0b4c48c565c --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_5; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.5"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyImpl.java new file mode 100644 index 00000000000..62d6597d682 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/TestLogImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/TestLogImpl.java new file mode 100644 index 00000000000..8cc7dec1f3b --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/VirtualRelationshipImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..0e6e2b491f3 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/build.gradle b/compatibility/5.5/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..330a0408d2c --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.5' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.5' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.5' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.5' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.5' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..177fe8e896b --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported55"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..0c98e19eeb7 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.5"; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCommandCreationContextImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..ac20edc223e --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.LATEST; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCountsStoreImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..8e6544b2055 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCountsStoreImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..8dec0e77ad9 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,185 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository55; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository55 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository55(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + byteOffset, + logVersion + ); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodeCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodeCursor.java new file mode 100644 index 00000000000..d6118867135 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodePropertyCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..3718d2ce791 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertyCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..48434b8e0a1 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertySelectionImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..146d3613887 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipPropertyCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..93fa5f6ba74 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipScanCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..42b15b04014 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipTraversalCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..66cb205b6f1 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..7c22c84e89a --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -0,0 +1,556 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory55; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-55"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_5, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker databaseReadOnlyChecker, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory55.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..1573e72d1d5 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineImpl.java @@ -0,0 +1,294 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader55; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private final StorageEngineIndexingBehaviour indexingBehaviour = () -> false; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return indexingBehaviour; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader55(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageLocksImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..122e557284b --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStoreVersion.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStoreVersion.java new file mode 100644 index 00000000000..000361016cf --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..0a6f3e8c889 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4] + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryVersionCheck.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryVersionCheck.java new file mode 100644 index 00000000000..1ae0d5d7f76 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._55.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..8c9a25a65ac --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_5; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.5"; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..7124a0829d1 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader55; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader55(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository55.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository55.java new file mode 100644 index 00000000000..cefa28655c4 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository55.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository55 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository55() { + this(0, 0); + } + + private InMemoryLogVersionRepository55(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory55.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory55.java new file mode 100644 index 00000000000..36530d653b5 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory55.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory55 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory55(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader55.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader55.java new file mode 100644 index 00000000000..67fd5e0c840 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader55.java @@ -0,0 +1,325 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._55.InMemoryNodeCursor; +import org.neo4j.gds.compat._55.InMemoryPropertyCursor; +import org.neo4j.gds.compat._55.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._55.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader55 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader55( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/build.gradle b/compatibility/5.6/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..28100243536 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.6' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.6' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.6' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.6' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.6' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.6' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.6' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..5f0279f2fc5 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.6 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.6 (placeholder)"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..98d6a61dca1 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.6 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.6 (placeholder)"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/BoltTransactionRunnerImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..974956d0662 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/BoltTransactionRunnerImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.bookmark.Bookmark; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.v41.message.request.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableProcedureImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableProcedureImpl.java new file mode 100644 index 00000000000..bd0b3c0a07a --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableUserAggregationFunctionImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..b1f07487e92 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatAccessModeImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatAccessModeImpl.java new file mode 100644 index 00000000000..230ae244460 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatGraphDatabaseAPIImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..b99d39e27c0 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatIndexQueryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..2e06af57725 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatUsernameAuthSubjectImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..3397d8d71a0 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompositeNodeCursorImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..b4d73fca9f6 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseLayoutImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..2173de12ab1 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..65ad6f6a250 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..f064a42e37c --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_6; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.6"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java new file mode 100644 index 00000000000..1458cdca72a --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -0,0 +1,935 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public long pageCacheMemory() { + return pageCacheMemory.orElseGet(Configuration.super::pageCacheMemory); + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/NodeLabelIndexLookupImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..0d9d2a3ae36 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/PartitionedStoreScan.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/PartitionedStoreScan.java new file mode 100644 index 00000000000..a3a52ce2266 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ReferencePropertyReference.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ReferencePropertyReference.java new file mode 100644 index 00000000000..303d8e528cc --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ScanBasedStoreScanImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..6da9397945f --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..af413d52556 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_6; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.6"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyImpl.java new file mode 100644 index 00000000000..ffd55552a5f --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/TestLogImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/TestLogImpl.java new file mode 100644 index 00000000000..e7ddb76d10e --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/VirtualRelationshipImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..84a53954543 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/build.gradle b/compatibility/5.6/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..e7ca6e2ff23 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.6' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.6' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.6' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.6' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.6' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java b/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..fc6e82cb270 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported56"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..5e6ab3d3d07 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.6"; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCommandCreationContextImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..2527d95468e --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCountsStoreImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..224369a1f17 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCountsStoreImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryMetaDataProviderImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..7460d90a9ca --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository56; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository56 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository56(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodeCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodeCursor.java new file mode 100644 index 00000000000..d1e9ac4939f --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodePropertyCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..b183e95d4eb --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertyCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..f4d8b3034d4 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertySelectionImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..d0c4d11bae8 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipPropertyCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..4b0895b48c7 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipScanCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..9efc1b9563d --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipTraversalCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..2b2ad14d00c --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..e489afc7173 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java @@ -0,0 +1,557 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory56; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-56"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_6, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory56.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..dd36aca80ee --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineImpl.java @@ -0,0 +1,294 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader56; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private final StorageEngineIndexingBehaviour indexingBehaviour = () -> false; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return indexingBehaviour; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader56(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageLocksImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..62bb86eb377 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStoreVersion.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStoreVersion.java new file mode 100644 index 00000000000..8f057a608a7 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryTransactionIdStoreImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..fb77020f557 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryVersionCheck.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryVersionCheck.java new file mode 100644 index 00000000000..e9a743e106f --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._56.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..4d03274003a --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_6; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.6"; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..b5a1b5a4ffc --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader56; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader56(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository56.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository56.java new file mode 100644 index 00000000000..0030425d089 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository56.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository56 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository56() { + this(0, 0); + } + + private InMemoryLogVersionRepository56(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory56.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory56.java new file mode 100644 index 00000000000..188dfd83144 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory56.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory56 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory56(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader56.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader56.java new file mode 100644 index 00000000000..921062f5176 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader56.java @@ -0,0 +1,325 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._56.InMemoryNodeCursor; +import org.neo4j.gds.compat._56.InMemoryPropertyCursor; +import org.neo4j.gds.compat._56.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._56.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader56 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader56( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/build.gradle b/compatibility/5.7/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..c80519fa5ac --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.7' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.7' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.7' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.7' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.7' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.7' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.7' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..48377e059a3 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.7 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.7 (placeholder)"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..6062d17edd2 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.7 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.7 (placeholder)"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/BoltTransactionRunnerImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..6a4b13ca2c1 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/BoltTransactionRunnerImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.bookmark.Bookmark; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableProcedureImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableProcedureImpl.java new file mode 100644 index 00000000000..9729407a4bd --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableUserAggregationFunctionImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..52a05063eb1 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatAccessModeImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatAccessModeImpl.java new file mode 100644 index 00000000000..7f009169ce0 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatGraphDatabaseAPIImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..6c9b26e3f2a --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatIndexQueryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..423c2908fd8 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatUsernameAuthSubjectImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..bf183d390c0 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompositeNodeCursorImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..968b165ee3b --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseLayoutImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..619c6359717 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..25ebefcf2b7 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..4b88ffcb482 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_7; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.7"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java new file mode 100644 index 00000000000..aee4d24f8ce --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java @@ -0,0 +1,930 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/NodeLabelIndexLookupImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..800f5875918 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/PartitionedStoreScan.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/PartitionedStoreScan.java new file mode 100644 index 00000000000..52263aec779 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ReferencePropertyReference.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ReferencePropertyReference.java new file mode 100644 index 00000000000..d7d4eb51c01 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ScanBasedStoreScanImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..13b6951eebd --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..a24ff5dc141 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_7; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.7"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyImpl.java new file mode 100644 index 00000000000..59d32e65627 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/TestLogImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/TestLogImpl.java new file mode 100644 index 00000000000..d6bf40e9819 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/VirtualRelationshipImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..96e1eae203a --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/build.gradle b/compatibility/5.7/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..d12c369acf3 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.7' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.7' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.7' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.7' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.7' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java b/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..ad053e0baee --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported57"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..76b6f44b528 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.7"; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCommandCreationContextImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..11f042220e0 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCountsStoreImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..3f6f64ecaa5 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryMetaDataProviderImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..42a8f5a3576 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository57; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository57 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository57(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodeCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodeCursor.java new file mode 100644 index 00000000000..c41c50fa13e --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodePropertyCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..1a18e5ff264 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertyCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..693f57da32b --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertySelectionImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..3ab5d56cc97 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipPropertyCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..34befce80f6 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipScanCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..332dd46668d --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipTraversalCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..7ac42153d33 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..cbace6c2601 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java @@ -0,0 +1,558 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory57; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-57"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_7, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory57.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..96b8f6a8ce9 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineImpl.java @@ -0,0 +1,294 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader57; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private final StorageEngineIndexingBehaviour indexingBehaviour = () -> false; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return indexingBehaviour; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader57(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageLocksImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..aa89bea2c72 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStoreVersion.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStoreVersion.java new file mode 100644 index 00000000000..3ab0f1cf00e --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryTransactionIdStoreImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..976995f6ad2 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryVersionCheck.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryVersionCheck.java new file mode 100644 index 00000000000..18ea10e4da9 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._57.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..67197cacc4e --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_7; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.7"; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..8a0eadab4b5 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader57; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader57(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository57.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository57.java new file mode 100644 index 00000000000..7fb6d12a43a --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository57.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository57 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository57() { + this(0, 0); + } + + private InMemoryLogVersionRepository57(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory57.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory57.java new file mode 100644 index 00000000000..02fdf497d40 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory57.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory57 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory57(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader57.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader57.java new file mode 100644 index 00000000000..1d64a8e97c1 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader57.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._57.InMemoryNodeCursor; +import org.neo4j.gds.compat._57.InMemoryPropertyCursor; +import org.neo4j.gds.compat._57.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._57.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader57 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader57( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/build.gradle b/compatibility/5.8/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..44bde179dab --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.8' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.8' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.8' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.8' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.8' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.8' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.8' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..2ddab471d98 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.8 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.8 (placeholder)"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..cf7d81502e2 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.8 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.8 (placeholder)"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/BoltTransactionRunnerImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..1e8ee114528 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableProcedureImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableProcedureImpl.java new file mode 100644 index 00000000000..3dfbea82de7 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableUserAggregationFunctionImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..bdcafd01e01 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatAccessModeImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatAccessModeImpl.java new file mode 100644 index 00000000000..97ed5b9e6f1 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatGraphDatabaseAPIImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..12d98ba7f9b --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatIndexQueryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..bdc9c206219 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatUsernameAuthSubjectImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..a3b2bdb802a --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompositeNodeCursorImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..8e6ccc18856 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseLayoutImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..7108a397ba9 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..edafd3e92dd --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..47ec85af7af --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_8; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.8"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java new file mode 100644 index 00000000000..1a8a9c75af6 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java @@ -0,0 +1,930 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/NodeLabelIndexLookupImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..13ea95378a2 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/PartitionedStoreScan.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/PartitionedStoreScan.java new file mode 100644 index 00000000000..b4a77f3bf84 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ReferencePropertyReference.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ReferencePropertyReference.java new file mode 100644 index 00000000000..9296154116d --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ScanBasedStoreScanImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..f3ae18f5fec --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..847538956a6 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_8; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.8"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyImpl.java new file mode 100644 index 00000000000..a6a5cafa274 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/TestLogImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/TestLogImpl.java new file mode 100644 index 00000000000..ba92ebb85eb --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/VirtualRelationshipImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..25287a363f6 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/build.gradle b/compatibility/5.8/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..e2d02c8f29b --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.8' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.8' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.8' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.8' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.8' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java b/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..19dbb9dfeac --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported58"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..ea12427918f --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.8"; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCommandCreationContextImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..abf59f71994 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCountsStoreImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..bc256458512 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryMetaDataProviderImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..c7eb156c0c6 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository58; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository58 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository58(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodeCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodeCursor.java new file mode 100644 index 00000000000..55857a73fc4 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodePropertyCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..079cda66b29 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertyCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..91f7dd25f1c --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertySelectionImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..43ce0c8d484 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipPropertyCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..4c3a5eb810b --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipScanCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..71cd9fcdc19 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipTraversalCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..5c82c1e02ac --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..36a4ff16c76 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java @@ -0,0 +1,558 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory58; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-58"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_8, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory58.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..f23b9764315 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineImpl.java @@ -0,0 +1,321 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader58; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + throw new UnsupportedOperationException(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader58(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageLocksImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..8ef6bfc8b53 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStoreVersion.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStoreVersion.java new file mode 100644 index 00000000000..37e1f46f25d --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryTransactionIdStoreImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..b607e28637e --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryVersionCheck.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryVersionCheck.java new file mode 100644 index 00000000000..187731326b5 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._58.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..c8f0a8393b8 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_8; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.8"; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..c47d09fd394 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader58; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader58(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository58.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository58.java new file mode 100644 index 00000000000..e495c490b52 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository58.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository58 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository58() { + this(0, 0); + } + + private InMemoryLogVersionRepository58(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory58.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory58.java new file mode 100644 index 00000000000..5b457388c12 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory58.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory58 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory58(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader58.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader58.java new file mode 100644 index 00000000000..3db83ec93cf --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader58.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._58.InMemoryNodeCursor; +import org.neo4j.gds.compat._58.InMemoryPropertyCursor; +import org.neo4j.gds.compat._58.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._58.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader58 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader58( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/build.gradle b/compatibility/5.9/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..e688f9dcf48 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.9' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.9' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.9' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..c972ffc94a3 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.9 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.9 (placeholder)"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..1a05f4b02fd --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.9 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.9 (placeholder)"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..479f8dbf179 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java new file mode 100644 index 00000000000..08ff067f966 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..de837dc4b2e --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java new file mode 100644 index 00000000000..49f92fb86e5 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..4fbf4bafb57 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..ccef737c591 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..85c75a5d377 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..3fc8fbfca62 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..01a2134ad14 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..e8e8185129c --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..25ad1de6d4e --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.9"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java new file mode 100644 index 00000000000..f87ab180270 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java @@ -0,0 +1,930 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..849d9a4d5cd --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java new file mode 100644 index 00000000000..2a8da868aeb --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java new file mode 100644 index 00000000000..a24847fc2de --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..c108e8bec61 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..c39ad8aa279 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.9"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java new file mode 100644 index 00000000000..662daf64e73 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java new file mode 100644 index 00000000000..7d09c9ce756 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..07708b93b25 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/build.gradle b/compatibility/5.9/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..17bd19ef579 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.9' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..4bdc7bd36f8 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported59"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..e43773a19d3 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.9"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..6bad2296b00 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..7cf4f836b7d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..77f9b72465f --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository59; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository59 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository59(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java new file mode 100644 index 00000000000..15d394b1586 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..43b8fd4837a --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..2b6499a1f96 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..5724e2829d3 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..806f0b25659 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..6f0c089ee8d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..6cd7ebf0310 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..ad9133ba7ea --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java @@ -0,0 +1,558 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory59; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-59"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_9, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory59.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock clock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..e52651080d1 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java @@ -0,0 +1,341 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader59; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + throw new UnsupportedOperationException(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader59(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory(StorageEngineFactory storageEngineFactory, Config config, SystemNanoClock clock) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..0b49cdb34aa --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java new file mode 100644 index 00000000000..fcd75bd3438 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..2e220e4d3bc --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java new file mode 100644 index 00000000000..fb196c18fbe --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._59.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..81132c0c9de --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.9"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..8cda8e608ed --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader59; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader59(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java new file mode 100644 index 00000000000..793eae1cabd --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository59 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository59() { + this(0, 0); + } + + private InMemoryLogVersionRepository59(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java new file mode 100644 index 00000000000..4aac7460e26 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory59 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory59(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java new file mode 100644 index 00000000000..b7e8ea9aae7 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._59.InMemoryNodeCursor; +import org.neo4j.gds.compat._59.InMemoryPropertyCursor; +import org.neo4j.gds.compat._59.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._59.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader59 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader59( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/api/neo4j-kernel-adapter/build.gradle b/compatibility/api/neo4j-kernel-adapter/build.gradle index db3367534e7..943fb9ef50e 100644 --- a/compatibility/api/neo4j-kernel-adapter/build.gradle +++ b/compatibility/api/neo4j-kernel-adapter/build.gradle @@ -13,6 +13,8 @@ dependencies { compileOnly group: 'org.immutables', name: 'builder', version: ver.'immutables' compileOnly group: 'org.jetbrains', name: 'annotations', version: ver.'jetbrains-annotations' compileOnly group: 'org.eclipse.collections', name: 'eclipse-collections', version: ver.'eclipse-collections' + compileOnly group: 'org.apache.logging.log4j', name: 'log4j-api', version: ver.'log4j' + neodeps().each { compileOnly(group: 'org.neo4j', name: it, version: ver.'neo4j') { transitive = false diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index 6843ffc68d4..ab22f31b4d4 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -39,6 +39,7 @@ import org.neo4j.internal.batchimport.input.Collector; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.ReadableGroups; import org.neo4j.internal.batchimport.staging.ExecutionMonitor; import org.neo4j.internal.helpers.HostnamePort; import org.neo4j.internal.id.IdGeneratorFactory; @@ -72,6 +73,9 @@ import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; @@ -80,6 +84,7 @@ import org.neo4j.procedure.Mode; import org.neo4j.scheduler.JobScheduler; import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.values.virtual.MapValue; import java.nio.file.Path; import java.util.List; @@ -107,6 +112,8 @@ long getHighestPossibleIdInUse( KernelTransaction kernelTransaction ); + long getHighId(RecordStore recordStore); + List> entityCursorScan( KernelTransaction transaction, int[] labelIds, @@ -204,9 +211,9 @@ BatchImporter instantiateBatchImporter( Input batchInputFrom(CompatInput compatInput); - InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType); + InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups); - InputEntityIdVisitor.String inputEntityStringIdVisitor(); + InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups); Setting additionalJvm(); @@ -309,4 +316,12 @@ UserFunctionSignature userFunctionSignature( void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext); + TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ); + + boolean isCompositeDatabase(GraphDatabaseService databaseService); } diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java index 6d9a8ab2191..e74f381612d 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java @@ -191,7 +191,6 @@ private static Neo4jVersionInfo loadNeo4jVersion() { .reason(e) .build() ) - .neo4jVersion(Neo4jVersion.V_Dev) .build(); } } diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxyApi.java index 8e044460110..831441d08f6 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxyApi.java @@ -31,4 +31,6 @@ public interface SettingProxyApi { @TestOnly void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService); + + String secondaryModeName(); } diff --git a/cypher/api/storage-engine-adapter/build.gradle b/compatibility/api/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/api/storage-engine-adapter/build.gradle rename to compatibility/api/storage-engine-adapter/build.gradle diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java similarity index 87% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java index 8245277e0c2..813163c80e9 100644 --- a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java +++ b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java @@ -35,8 +35,22 @@ import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; +import java.util.Locale; + public interface StorageEngineProxyApi { + static void requireNeo4jVersion(Neo4jVersion version, Class self) { + if (Neo4jVersion.findNeo4jVersion() != version) { + throw new IllegalStateException(String.format( + Locale.ENGLISH, + "This '%s' instance is only compatible with Neo4j version %s, but Neo4j version %s has been detected.", + self.getName(), + version, + Neo4jVersion.findNeo4jVersion() + )); + } + } + // InMemoryStorageEngineBuilder inMemoryStorageEngineBuilder( // DatabaseLayout databaseLayout, // TokenHolders tokenHolders diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java diff --git a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java new file mode 100644 index 00000000000..589ae4775b3 --- /dev/null +++ b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.util.concurrent.ArrayQueueOutOfOrderSequence; +import org.neo4j.util.concurrent.OutOfOrderSequence; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +public abstract class AbstractTransactionIdStore implements TransactionIdStore { + protected final AtomicLong committingTransactionId; + protected final OutOfOrderSequence closedTransactionId; + protected final AtomicReference committedTransactionId; + protected final long previouslyCommittedTxId; + protected final int initialTransactionChecksum; + protected final long previouslyCommittedTxCommitTimestamp; + + public AbstractTransactionIdStore() { + this(1L, -559063315, 0L, 0L, 64L); + } + + public AbstractTransactionIdStore( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogVersion, + long previouslyCommittedTxLogByteOffset + ) { + this.committingTransactionId = new AtomicLong(); + this.closedTransactionId = new ArrayQueueOutOfOrderSequence(-1L, 100, new long[5]); + this.committedTransactionId = new AtomicReference<>(transactionId(1L, -559063315, 0L)); + + assert previouslyCommittedTxId >= 1L : "cannot start from a tx id less than BASE_TX_ID"; + + initLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogVersion, + previouslyCommittedTxLogByteOffset + ); + this.previouslyCommittedTxId = previouslyCommittedTxId; + this.initialTransactionChecksum = checksum; + this.previouslyCommittedTxCommitTimestamp = previouslyCommittedTxCommitTimestamp; + } + + protected abstract void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ); + + protected abstract TransactionId transactionId(long transactionId, int checksum, long commitTimestamp); + + @Override + public long nextCommittingTransactionId() { + return this.committingTransactionId.incrementAndGet(); + } + + @Override + public long committingTransactionId() { + return this.committingTransactionId.get(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.committedTransactionId.get().transactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.committedTransactionId.get(); + } + + @Override + public long getLastClosedTransactionId() { + return this.closedTransactionId.getHighestGapFreeNumber(); + } +} diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index 92a01719d11..8a2490bb865 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -39,6 +39,7 @@ import org.neo4j.internal.batchimport.input.Collector; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.ReadableGroups; import org.neo4j.internal.batchimport.staging.ExecutionMonitor; import org.neo4j.internal.helpers.HostnamePort; import org.neo4j.internal.id.IdGeneratorFactory; @@ -72,6 +73,9 @@ import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; @@ -80,6 +84,7 @@ import org.neo4j.procedure.Mode; import org.neo4j.scheduler.JobScheduler; import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.values.virtual.MapValue; import java.nio.file.Path; import java.util.List; @@ -126,6 +131,10 @@ public static long getHighestPossibleIdInUse( return IMPL.getHighestPossibleIdInUse(recordStore, kernelTransaction); } + public static long getHighId(RecordStore recordStore) { + return IMPL.getHighId(recordStore); + }; + public static List> entityCursorScan( KernelTransaction transaction, int[] labelIds, @@ -276,12 +285,12 @@ public static BatchImporter instantiateBatchImporter( ); } - public static InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { - return IMPL.inputEntityLongIdVisitor(idType); + public static InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + return IMPL.inputEntityLongIdVisitor(idType, groups); } - public static InputEntityIdVisitor.String inputEntityStringIdVisitor() { - return IMPL.inputEntityStringIdVisitor(); + public static InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + return IMPL.inputEntityStringIdVisitor(groups); } public static Input batchInputFrom(CompatInput compatInput) { @@ -466,6 +475,18 @@ public static long transactionId(KernelTransactionHandle kernelTransactionHandle public static void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { IMPL.reserveNeo4jIds(generatorFactory, size, cursorContext); } + public static TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return IMPL.newQueryContext(contextFactory, tx, queryText, queryParameters); + } + + public static boolean isCompositeDatabase(GraphDatabaseService databaseService) { + return IMPL.isCompositeDatabase(databaseService); + } private Neo4jProxy() { throw new UnsupportedOperationException("No instances"); diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxy.java index f046a796697..73e83753ea4 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxy.java @@ -49,5 +49,9 @@ public static void setDatabaseMode(Config config, DatabaseMode databaseMode, Gra IMPL.setDatabaseMode(config, databaseMode, databaseService); } + public static String secondaryModeName() { + return IMPL.secondaryModeName(); + } + private SettingProxy() {} } diff --git a/cypher/common/storage-engine-adapter/build.gradle b/compatibility/common/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/common/storage-engine-adapter/build.gradle rename to compatibility/common/storage-engine-adapter/build.gradle diff --git a/cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java similarity index 100% rename from cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java rename to compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java diff --git a/cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java similarity index 100% rename from cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java rename to compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java diff --git a/cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java similarity index 100% rename from cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java rename to compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java diff --git a/config-api/src/main/java/org/neo4j/gds/config/BaseConfig.java b/config-api/src/main/java/org/neo4j/gds/config/BaseConfig.java index f6124f16b88..1a00b715862 100644 --- a/config-api/src/main/java/org/neo4j/gds/config/BaseConfig.java +++ b/config-api/src/main/java/org/neo4j/gds/config/BaseConfig.java @@ -32,6 +32,7 @@ public interface BaseConfig extends ToMapConvertible { String SUDO_KEY = "sudo"; + String LOG_PROGRESS_KEY = "logProgress"; @Value.Parameter(false) @Configuration.Key("username") @@ -45,6 +46,13 @@ default boolean sudo() { return false; } + @Value.Default + @Value.Parameter(false) + @Configuration.Key(LOG_PROGRESS_KEY) + default boolean logProgress() { + return true; + } + @Configuration.CollectKeys @Value.Auxiliary @Value.Default diff --git a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java index 11fc8d4d9f2..315d5d91dfc 100644 --- a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.api; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.loading.CSRGraphStore; @@ -27,8 +27,11 @@ import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.mem.MemoryUsage; +import java.util.Map; + import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public abstract class CSRGraphStoreFactory extends GraphStoreFactory { @@ -42,15 +45,18 @@ public CSRGraphStoreFactory( super(graphProjectConfig, capabilities, loadingContext, dimensions); } - protected CSRGraphStore createGraphStore( - Nodes nodes, - RelationshipImportResult relationshipImportResult - ) { + protected CSRGraphStore createGraphStore(Nodes nodes, RelationshipImportResult relationshipImportResult) { + var schema = MutableGraphSchema.of( + nodes.schema(), + relationshipImportResult.relationshipSchema(), + Map.of() + ); + return new GraphStoreBuilder() .databaseId(DatabaseId.of(loadingContext.graphDatabaseService())) .capabilities(capabilities) - .schema(computeGraphSchema(nodes, relationshipImportResult)) - .nodes(Nodes.of(nodes.idMap(), nodes.properties())) + .schema(schema) + .nodes(nodes) .relationshipImportResult(relationshipImportResult) .concurrency(graphProjectConfig.readConcurrency()) .build(); @@ -58,12 +64,11 @@ protected CSRGraphStore createGraphStore( protected void logLoadingSummary(GraphStore graphStore) { var sizeInBytes = MemoryUsage.sizeOf(graphStore); - var memoryUsage = MemoryUsage.humanReadable(sizeInBytes); - progressTracker.logInfo(formatWithLocale("Actual memory usage of the loaded graph: %s", memoryUsage)); + if (sizeInBytes >= 0) { + var memoryUsage = MemoryUsage.humanReadable(sizeInBytes); + progressTracker().logInfo(formatWithLocale("Actual memory usage of the loaded graph: %s", memoryUsage)); + } } - protected abstract GraphSchema computeGraphSchema( - Nodes nodes, - RelationshipImportResult relationshipImportResult - ); + protected abstract ProgressTracker progressTracker(); } diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStore.java b/core/src/main/java/org/neo4j/gds/api/GraphStore.java index a59b9e2895c..e82d712445e 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStore.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStore.java @@ -163,10 +163,7 @@ default Collection relationshipPropertyKeys(Collection RelationshipProperty relationshipPropertyValues(RelationshipType relationshipType, String propertyKey); - void addRelationshipType( - RelationshipType relationshipType, - SingleTypeRelationships relationships - ); + void addRelationshipType(SingleTypeRelationships relationships); void addInverseIndex( RelationshipType relationshipType, diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java b/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java index 80dfbdcf22f..f0f978e8323 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java @@ -230,11 +230,8 @@ public RelationshipProperty relationshipPropertyValues( } @Override - public void addRelationshipType( - RelationshipType relationshipType, - SingleTypeRelationships relationships - ) { - graphStore.addRelationshipType(relationshipType, relationships); + public void addRelationshipType(SingleTypeRelationships relationships) { + graphStore.addRelationshipType(relationships); } @Override diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/GraphStoreFactory.java index b77261b1af3..3ac0a775704 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStoreFactory.java @@ -24,7 +24,6 @@ import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.gds.core.utils.mem.MemoryEstimation; -import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; /** * The Abstract Factory defines the construction of the graph @@ -46,7 +45,6 @@ public interface Supplier { protected final Capabilities capabilities; protected final GraphLoaderContext loadingContext; protected final GraphDimensions dimensions; - protected final ProgressTracker progressTracker; public GraphStoreFactory( CONFIG graphProjectConfig, @@ -58,7 +56,6 @@ public GraphStoreFactory( this.capabilities = capabilities; this.loadingContext = loadingContext; this.dimensions = dimensions; - this.progressTracker = initProgressTracker(); } public abstract STORE build(); @@ -79,8 +76,6 @@ public CONFIG graphProjectConfig() { return graphProjectConfig; } - protected abstract ProgressTracker initProgressTracker(); - @ValueClass public interface ImportResult { STORE graphStore(); diff --git a/core/src/main/java/org/neo4j/gds/api/properties/PropertyValues.java b/core/src/main/java/org/neo4j/gds/api/properties/PropertyValues.java index 6f8183c97dc..607e6d0fa1b 100644 --- a/core/src/main/java/org/neo4j/gds/api/properties/PropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/api/properties/PropertyValues.java @@ -36,11 +36,6 @@ default long release() { return 0; } - /** - * @return the number of values stored. - */ - long size(); - default UnsupportedOperationException unsupportedTypeException(ValueType expectedType) { return new UnsupportedOperationException(formatWithLocale("Tried to retrieve a value of type %s value from properties of type %s", expectedType, valueType())); } diff --git a/core/src/main/java/org/neo4j/gds/api/properties/graph/GraphPropertyValues.java b/core/src/main/java/org/neo4j/gds/api/properties/graph/GraphPropertyValues.java index 8c9a35a02e5..bc5a3007369 100644 --- a/core/src/main/java/org/neo4j/gds/api/properties/graph/GraphPropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/api/properties/graph/GraphPropertyValues.java @@ -52,4 +52,6 @@ default Stream longArrayValues() { Stream objects(); Stream values(); + + long valueCount(); } diff --git a/core/src/main/java/org/neo4j/gds/api/properties/nodes/DoubleNodePropertyValues.java b/core/src/main/java/org/neo4j/gds/api/properties/nodes/DoubleNodePropertyValues.java index 30f414b5167..d61824abfd4 100644 --- a/core/src/main/java/org/neo4j/gds/api/properties/nodes/DoubleNodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/api/properties/nodes/DoubleNodePropertyValues.java @@ -50,7 +50,7 @@ default ValueType valueType() { @Override default OptionalDouble getMaxDoublePropertyValue() { return LongStream - .range(0, size()) + .range(0, nodeCount()) .parallel() .mapToDouble(this::doubleValue) .max(); diff --git a/core/src/main/java/org/neo4j/gds/api/properties/nodes/LongNodePropertyValues.java b/core/src/main/java/org/neo4j/gds/api/properties/nodes/LongNodePropertyValues.java index 68413e40f73..9d1ff7b9e37 100644 --- a/core/src/main/java/org/neo4j/gds/api/properties/nodes/LongNodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/api/properties/nodes/LongNodePropertyValues.java @@ -60,7 +60,7 @@ default double doubleValue(long nodeId) { @Override default OptionalLong getMaxLongPropertyValue() { return LongStream - .range(0, size()) + .range(0, nodeCount()) .parallel() .map(this::longValue) .max(); diff --git a/core/src/main/java/org/neo4j/gds/api/properties/nodes/NodePropertyValues.java b/core/src/main/java/org/neo4j/gds/api/properties/nodes/NodePropertyValues.java index 93c9c3778d0..5d89a9f0130 100644 --- a/core/src/main/java/org/neo4j/gds/api/properties/nodes/NodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/api/properties/nodes/NodePropertyValues.java @@ -59,6 +59,8 @@ default long[] longArrayValue(long nodeId) { Value value(long nodeId); + long nodeCount(); + /** * @return the maximum long value contained in the mapping or an empty {@link OptionalLong} if the mapping is * empty or the feature is not supported. diff --git a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java index 2f189684fa4..6b4d81ff0ba 100644 --- a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java +++ b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java @@ -27,8 +27,9 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.config.RandomGraphGeneratorConfig.AllowSelfLoops; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.huge.HugeGraph; @@ -127,6 +128,7 @@ public HugeGraph generate() { var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(relationshipType) .orientation(direction.toOrientation()) .addAllPropertyConfigs(maybeRelationshipPropertyProducer .map(propertyProducer -> List.of(GraphFactory.PropertyConfig.of( @@ -143,9 +145,12 @@ public HugeGraph generate() { var relationships = relationshipsBuilder.build(); - var graphSchema = GraphSchema.of( + var relationshipSchema = MutableRelationshipSchema.empty(); + relationshipSchema.set(relationships.relationshipSchemaEntry()); + + var graphSchema = MutableGraphSchema.of( nodePropertiesAndSchema.nodeSchema(), - relationships.relationshipSchema(relationshipType), + relationshipSchema, Map.of() ); @@ -226,14 +231,14 @@ private void addDagRelationship(RelationshipsBuilder relationshipsImporter, @ValueClass interface NodePropertiesAndSchema { - NodeSchema nodeSchema(); + MutableNodeSchema nodeSchema(); Map nodeProperties(); } private NodePropertiesAndSchema generateNodeProperties(IdMap idMap) { if (this.nodePropertyProducers.isEmpty()) { - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); idMap.availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); return ImmutableNodePropertiesAndSchema.builder() .nodeSchema(nodeSchema) @@ -284,7 +289,7 @@ private NodePropertiesAndSchema generateNodeProperties(IdMap idMap) { )); // Create a corresponding node schema - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); generatedProperties.forEach((propertyKey, property) -> propertyNameToLabels .get(propertyKey) .forEach(nodeLabel -> { diff --git a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromCypherConfig.java b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromCypherConfig.java index 22142f0b52a..876a8daeb8c 100644 --- a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromCypherConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromCypherConfig.java @@ -80,14 +80,14 @@ default GraphStoreFactory.Supplier graphStoreFactory() { return new GraphStoreFactory.Supplier() { @Override public GraphStoreFactory get(GraphLoaderContext loaderContext) { - return new CypherFactory(GraphProjectFromCypherConfig.this, loaderContext); + return CypherFactory.createWithDerivedDimensions(GraphProjectFromCypherConfig.this, loaderContext); } @Override public GraphStoreFactory getWithDimension( GraphLoaderContext loaderContext, GraphDimensions graphDimensions ) { - return new CypherFactory(GraphProjectFromCypherConfig.this, loaderContext, graphDimensions); + return CypherFactory.createWithBaseDimensions(GraphProjectFromCypherConfig.this, loaderContext, graphDimensions); } }; } diff --git a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java index ca75eeab5d2..1b001c86356 100644 --- a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java @@ -52,8 +52,8 @@ public interface GraphProjectFromStoreConfig extends GraphProjectConfig { String RELATIONSHIP_PROPERTIES_KEY = "relationshipProperties"; @Key(NODE_PROJECTION_KEY) - @ConvertWith(method = "org.neo4j.gds.AbstractNodeProjections#fromObject") - @Configuration.ToMapValue("org.neo4j.gds.AbstractNodeProjections#toObject") + @ConvertWith(method = "org.neo4j.gds.NodeProjections#fromObject") + @Configuration.ToMapValue("org.neo4j.gds.NodeProjections#toObject") NodeProjections nodeProjections(); @Key(RELATIONSHIP_PROJECTION_KEY) @@ -63,16 +63,16 @@ public interface GraphProjectFromStoreConfig extends GraphProjectConfig { @Value.Default @Value.Parameter(false) - @Configuration.ConvertWith(method = "org.neo4j.gds.AbstractPropertyMappings#fromObject") - @Configuration.ToMapValue("org.neo4j.gds.AbstractPropertyMappings#toObject") + @Configuration.ConvertWith(method = "org.neo4j.gds.PropertyMappings#fromObject") + @Configuration.ToMapValue("org.neo4j.gds.PropertyMappings#toObject") default PropertyMappings nodeProperties() { return PropertyMappings.of(); } @Value.Default @Value.Parameter(false) - @Configuration.ConvertWith(method = "org.neo4j.gds.AbstractPropertyMappings#fromObject") - @Configuration.ToMapValue("org.neo4j.gds.AbstractPropertyMappings#toObject") + @Configuration.ConvertWith(method = "org.neo4j.gds.PropertyMappings#fromObject") + @Configuration.ToMapValue("org.neo4j.gds.PropertyMappings#toObject") default PropertyMappings relationshipProperties() { return PropertyMappings.of(); } diff --git a/core/src/main/java/org/neo4j/gds/config/GraphStreamRelationshipsConfig.java b/core/src/main/java/org/neo4j/gds/config/GraphStreamRelationshipsConfig.java index 720abc3cf6a..584ffff1cd9 100644 --- a/core/src/main/java/org/neo4j/gds/config/GraphStreamRelationshipsConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/GraphStreamRelationshipsConfig.java @@ -23,7 +23,6 @@ import org.neo4j.gds.ElementProjection; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.core.CypherMapWrapper; @@ -36,9 +35,8 @@ import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -@ValueClass @Configuration -public interface GraphStreamRelationshipsConfig extends BaseConfig { +public interface GraphStreamRelationshipsConfig extends BaseConfig, ConcurrencyConfig { @Configuration.Parameter Optional graphName(); @@ -49,11 +47,6 @@ default List relationshipTypes() { return Collections.singletonList(ElementProjection.PROJECT_ALL); } - @Value.Default - default int concurrency() { - return ConcurrencyConfig.DEFAULT_CONCURRENCY; - } - @Configuration.Ignore default Collection relationshipTypeIdentifiers(GraphStore graphStore) { return relationshipTypes().contains(ElementProjection.PROJECT_ALL) diff --git a/core/src/main/java/org/neo4j/gds/config/RandomGraphGeneratorConfig.java b/core/src/main/java/org/neo4j/gds/config/RandomGraphGeneratorConfig.java index a6bd7c8582f..3f4e211c731 100644 --- a/core/src/main/java/org/neo4j/gds/config/RandomGraphGeneratorConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/RandomGraphGeneratorConfig.java @@ -21,6 +21,7 @@ import org.immutables.value.Value; import org.jetbrains.annotations.Nullable; +import org.neo4j.gds.ImmutableNodeProjections; import org.neo4j.gds.ImmutableRelationshipProjections; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; @@ -97,9 +98,9 @@ default Map relationshipProperty() { } @Value.Default - @Configuration.ToMapValue("org.neo4j.gds.AbstractNodeProjections#toObject") + @Configuration.ToMapValue("org.neo4j.gds.NodeProjections#toObject") default NodeProjections nodeProjections() { - return NodeProjections.builder() + return ImmutableNodeProjections.builder() .putProjection( NodeLabel.of(nodeCount() + "_Nodes"), NodeProjection.of(nodeCount() + "_Nodes")) diff --git a/core/src/main/java/org/neo4j/gds/core/GdsLogExtension.java b/core/src/main/java/org/neo4j/gds/core/GdsLogExtension.java new file mode 100644 index 00000000000..72de2c3f8e2 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/GdsLogExtension.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.kernel.extension.ExtensionFactory; +import org.neo4j.kernel.extension.ExtensionType; +import org.neo4j.kernel.extension.context.ExtensionContext; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.logging.internal.LogService; + +@ServiceProvider +public class GdsLogExtension extends ExtensionFactory { + + public GdsLogExtension() { + super(ExtensionType.GLOBAL, "gds.log"); + } + + @Override + public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies) { + return LifecycleAdapter.onInit(() -> { + var log = Neo4jProxy.getUserLog(dependencies.logService(), GdsLogExtension.class); + GraphStoreCatalog.setLog(log); + }); + } + + interface Dependencies { + LogService logService(); + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/GraphDimensionsCypherReader.java b/core/src/main/java/org/neo4j/gds/core/GraphDimensionsCypherReader.java deleted file mode 100644 index f3127f4da07..00000000000 --- a/core/src/main/java/org/neo4j/gds/core/GraphDimensionsCypherReader.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.core; - -import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.NodeProjections; -import org.neo4j.gds.RelationshipProjections; -import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.config.GraphProjectFromCypherConfig; -import org.neo4j.gds.transaction.TransactionContext; -import org.neo4j.internal.id.IdGeneratorFactory; -import org.neo4j.internal.kernel.api.TokenRead; - -import static org.neo4j.gds.core.GraphDimensions.ANY_LABEL; -import static org.neo4j.gds.core.GraphDimensions.ANY_RELATIONSHIP_TYPE; - -public class GraphDimensionsCypherReader extends GraphDimensionsReader { - - public GraphDimensionsCypherReader( - TransactionContext tx, - GraphProjectFromCypherConfig config, - IdGeneratorFactory idGeneratorFactory - ) { - super(tx, config, idGeneratorFactory); - } - - @Override - protected TokenElementIdentifierMappings getNodeLabelTokens(TokenRead tokenRead) { - return new TokenElementIdentifierMappings<>(ANY_LABEL); - } - - @Override - protected TokenElementIdentifierMappings getRelationshipTypeTokens(TokenRead tokenRead) { - return new TokenElementIdentifierMappings<>(ANY_RELATIONSHIP_TYPE); - } - - @Override - protected NodeProjections getNodeProjections() { - return NodeProjections.all(); - } - - @Override - protected RelationshipProjections getRelationshipProjections() { - return RelationshipProjections.ALL; - } -} diff --git a/core/src/main/java/org/neo4j/gds/core/huge/FilteredNodePropertyValues.java b/core/src/main/java/org/neo4j/gds/core/huge/FilteredNodePropertyValues.java index ff292d689f8..1be2bdec92e 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/FilteredNodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/FilteredNodePropertyValues.java @@ -141,10 +141,9 @@ public long release() { graph = null; return releasedFromProps; } - @Override - public long size() { - return Math.min(properties.size(), graph.nodeCount()); + public long nodeCount() { + return graph.nodeCount(); } // This class is used when the ID space of the wrapped properties is wider than the id space used to retrieved node properties. diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java index 6f226f362ac..04787cc2158 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java @@ -61,8 +61,8 @@ public class ArrayIdMap extends LabeledIdMap { private final long highestNeoId; - private final HugeLongArray graphIds; - private final HugeSparseLongArray nodeToGraphIds; + private final HugeLongArray internalToOriginalIds; + private final HugeSparseLongArray originalToInternalIds; public static MemoryEstimation memoryEstimation() { return ESTIMATION; @@ -72,26 +72,26 @@ public static MemoryEstimation memoryEstimation() { * initialize the map with pre-built sub arrays */ public ArrayIdMap( - HugeLongArray graphIds, - HugeSparseLongArray nodeToGraphIds, + HugeLongArray internalToOriginalIds, + HugeSparseLongArray originalToInternalIds, LabelInformation labelInformation, long nodeCount, long highestNeoId ) { super(labelInformation, nodeCount); - this.graphIds = graphIds; - this.nodeToGraphIds = nodeToGraphIds; + this.internalToOriginalIds = internalToOriginalIds; + this.originalToInternalIds = originalToInternalIds; this.highestNeoId = highestNeoId; } @Override public long toMappedNodeId(long originalNodeId) { - return nodeToGraphIds.get(originalNodeId); + return originalToInternalIds.get(originalNodeId); } @Override public long toOriginalNodeId(long mappedNodeId) { - return graphIds.get(mappedNodeId); + return internalToOriginalIds.get(mappedNodeId); } @Override @@ -106,7 +106,7 @@ public IdMap rootIdMap() { @Override public boolean contains(final long originalNodeId) { - return nodeToGraphIds.contains(originalNodeId); + return originalToInternalIds.contains(originalNodeId); } @Override @@ -141,7 +141,7 @@ public Optional withFilteredLabels(Collection nodeLabe HugeSparseLongArray newNodeToGraphIds = ArrayIdMapBuilderOps.buildSparseIdMap( newNodeCount, - nodeToGraphIds.capacity(), + originalToInternalIds.capacity(), concurrency, newGraphIds ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java index 28d1357a957..c31e30aa206 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java @@ -69,8 +69,8 @@ public IdMap build( ) { adders.close(); long nodeCount = this.size(); - var graphIds = this.array(); - return ArrayIdMapBuilderOps.build(graphIds, nodeCount, labelInformationBuilder, highestNodeId, concurrency); + HugeLongArray internalToOriginalIds = this.array(); + return ArrayIdMapBuilderOps.build(internalToOriginalIds, nodeCount, labelInformationBuilder, highestNodeId, concurrency); } public HugeLongArray array() { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java index ba44e264b4a..cdc281241c8 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java @@ -31,28 +31,28 @@ public final class ArrayIdMapBuilderOps { static ArrayIdMap build( - HugeLongArray graphIds, + HugeLongArray internalToOriginalIds, long nodeCount, LabelInformation.Builder labelInformationBuilder, long highestNodeId, int concurrency ) { if (highestNodeId == NodesBuilder.UNKNOWN_MAX_ID) { - highestNodeId = graphIds.asNodeProperties().getMaxLongPropertyValue().orElse(NodesBuilder.UNKNOWN_MAX_ID); + highestNodeId = internalToOriginalIds.asNodeProperties().getMaxLongPropertyValue().orElse(NodesBuilder.UNKNOWN_MAX_ID); } - HugeSparseLongArray nodeToGraphIds = buildSparseIdMap( + HugeSparseLongArray originalToInternalIds = buildSparseIdMap( nodeCount, highestNodeId, concurrency, - graphIds + internalToOriginalIds ); - var labelInformation = labelInformationBuilder.build(nodeCount, nodeToGraphIds::get); + var labelInformation = labelInformationBuilder.build(nodeCount, originalToInternalIds::get); return new ArrayIdMap( - graphIds, - nodeToGraphIds, + internalToOriginalIds, + originalToInternalIds, labelInformation, nodeCount, highestNodeId diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java index ef302014153..e5781839b3b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java @@ -46,9 +46,10 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.huge.CSRCompositeRelationshipIterator; import org.neo4j.gds.core.huge.HugeGraphBuilder; import org.neo4j.gds.core.huge.NodeFilteredGraph; @@ -90,7 +91,7 @@ public class CSRGraphStore implements GraphStore { private final Set createdGraphs; - private GraphSchema schema; + private MutableGraphSchema schema; private GraphPropertyStore graphProperties; @@ -102,7 +103,7 @@ public class CSRGraphStore implements GraphStore { public static CSRGraphStore of( DatabaseId databaseId, Capabilities capabilities, - GraphSchema schema, + MutableGraphSchema schema, Nodes nodes, RelationshipImportResult relationshipImportResult, Optional graphProperties, @@ -123,7 +124,7 @@ public static CSRGraphStore of( private CSRGraphStore( DatabaseId databaseId, Capabilities capabilities, - GraphSchema schema, + MutableGraphSchema schema, IdMap nodes, NodePropertyStore nodeProperties, Map relationships, @@ -213,7 +214,7 @@ public void addGraphProperty(String propertyKey, GraphPropertyValues propertyVal var newGraphPropertySchema = new HashMap<>(schema().graphProperties()); newGraphPropertySchema.put(propertyKey, PropertySchema.of(propertyKey, propertyValues.valueType())); - this.schema = GraphSchema.of(schema().nodeSchema(), schema().relationshipSchema(), newGraphPropertySchema); + this.schema = MutableGraphSchema.of(schema.nodeSchema(), schema.relationshipSchema(), newGraphPropertySchema); }); } @@ -229,7 +230,7 @@ public void removeGraphProperty(String propertyKey) { var newGraphPropertySchema = new HashMap<>(schema().graphProperties()); newGraphPropertySchema.remove(propertyKey); - this.schema = GraphSchema.of(schema().nodeSchema(), schema().relationshipSchema(), newGraphPropertySchema); + this.schema = MutableGraphSchema.of(schema.nodeSchema(), schema.relationshipSchema(), newGraphPropertySchema); }); } @@ -370,24 +371,14 @@ public boolean hasRelationshipProperty(RelationshipType relType, String property @Override public ValueType relationshipPropertyType(String propertyKey) { - return relationships - .values() - .stream() - .flatMap(relationship -> relationship.properties().stream()) - .filter(propertyStore -> propertyStore.containsKey(propertyKey)) - .map(propertyStore -> propertyStore.get(propertyKey).valueType()) - .findFirst() + return Optional.ofNullable(schema().relationshipSchema().unionProperties().get(propertyKey)) + .map(PropertySchema::valueType) .orElse(ValueType.UNKNOWN); } @Override public Set relationshipPropertyKeys() { - return relationships - .values() - .stream() - .flatMap(relationship -> relationship.properties().stream()) - .flatMap(propertyStore -> propertyStore.keySet().stream()) - .collect(Collectors.toSet()); + return schema().relationshipSchema().allProperties(); } @Override @@ -409,15 +400,10 @@ public RelationshipProperty relationshipPropertyValues(RelationshipType relation } @Override - public void addRelationshipType( - RelationshipType relationshipType, SingleTypeRelationships relationships - ) { + public void addRelationshipType(SingleTypeRelationships relationships) { updateGraphStore(graphStore -> { - graphStore.relationships.computeIfAbsent(relationshipType, __ -> { - var relationshipSchemaEntry = schema() - .relationshipSchema() - .getOrCreateRelationshipType(relationshipType, relationships.direction()); - relationships.updateRelationshipSchemaEntry(relationshipSchemaEntry); + graphStore.relationships.computeIfAbsent(relationships.relationshipSchemaEntry().identifier(), __ -> { + schema.relationshipSchema().set(relationships.relationshipSchemaEntry()); return relationships; }); }); @@ -452,7 +438,7 @@ public DeletionResult deleteRelationships(RelationshipType relationshipType) { property.values().elementCount() )); }); - schema().relationshipSchema().remove(relationshipType); + schema.relationshipSchema().remove(relationshipType); }, () -> builder.deletedRelationships(0)); })); } @@ -598,7 +584,7 @@ private CSRGraph createGraph( ) { var filteredNodes = getFilteredIdMap(nodeLabels); Map filteredNodeProperties = filterNodeProperties(nodeLabels); - var nodeSchema = schema().nodeSchema().filter(new HashSet<>(nodeLabels)); + var nodeSchema = schema.nodeSchema().filter(new HashSet<>(nodeLabels)); return createGraphFromRelationshipType(filteredNodes, filteredNodeProperties, nodeSchema, @@ -614,7 +600,7 @@ private CSRGraph createGraph( ) { var filteredNodes = getFilteredIdMap(filteredLabels); Map filteredNodeProperties = filterNodeProperties(filteredLabels); - var nodeSchema = schema().nodeSchema().filter(new HashSet<>(filteredLabels)); + var nodeSchema = schema.nodeSchema().filter(new HashSet<>(filteredLabels)); List filteredGraphs = relationships .keySet() @@ -636,9 +622,9 @@ private CSRGraph createGraph( private CSRGraph createNodeOnlyGraph(Collection nodeLabels) { var filteredNodes = getFilteredIdMap(nodeLabels); var filteredNodeProperties = filterNodeProperties(nodeLabels); - var nodeSchema = schema().nodeSchema().filter(new HashSet<>(nodeLabels)); + var nodeSchema = schema.nodeSchema().filter(new HashSet<>(nodeLabels)); - var graphSchema = GraphSchema.of(nodeSchema, RelationshipSchema.empty(), schema.graphProperties()); + var graphSchema = MutableGraphSchema.of(nodeSchema, MutableRelationshipSchema.empty(), schema.graphProperties()); var initialGraph = new HugeGraphBuilder() .nodes(nodes) @@ -663,12 +649,12 @@ private Optional getFilteredIdMap(Collection private CSRGraph createGraphFromRelationshipType( Optional filteredNodes, Map filteredNodeProperties, - NodeSchema nodeSchema, + MutableNodeSchema nodeSchema, RelationshipType relationshipType, Optional maybeRelationshipProperty ) { - var graphSchema = GraphSchema.of(nodeSchema, - schema().relationshipSchema().filter(Set.of(relationshipType)), + var graphSchema = MutableGraphSchema.of(nodeSchema, + schema.relationshipSchema().filter(Set.of(relationshipType)), schema.graphProperties() ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index 7915d27398c..769f50b8c64 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -20,33 +20,24 @@ package org.neo4j.gds.core.loading; import org.jetbrains.annotations.NotNull; -import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.Properties; -import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.RelationshipProperty; import org.neo4j.gds.api.RelationshipPropertyStore; import org.neo4j.gds.api.ValueTypes; -import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.properties.graph.GraphPropertyStore; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; -import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.PropertySchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.huge.HugeGraph; +import org.neo4j.gds.utils.StringJoining; import org.neo4j.values.storable.NumberType; -import java.util.Collection; import java.util.Map; import java.util.Optional; -import java.util.function.Function; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -55,16 +46,9 @@ public final class CSRGraphStoreUtil { public static CSRGraphStore createFromGraph( DatabaseId databaseId, HugeGraph graph, - String relationshipTypeString, Optional relationshipPropertyKey, int concurrency ) { - var relationshipType = RelationshipType.of(relationshipTypeString); - Direction direction = graph.schema().isUndirected() ? Direction.UNDIRECTED : Direction.DIRECTED; - - var relationshipSchema = RelationshipSchema.empty(); - var entry = relationshipSchema.getOrCreateRelationshipType(relationshipType, direction); - relationshipPropertyKey.ifPresent(property -> { if (!graph.hasRelationshipProperty()) { @@ -73,40 +57,49 @@ public static CSRGraphStore createFromGraph( property )); } - - entry.addProperty( - property, - ValueType.DOUBLE, - PropertyState.PERSISTENT - ); }); + var schema = MutableGraphSchema.from(graph.schema()); + var relationshipSchema = schema.relationshipSchema(); + + if (relationshipSchema.availableTypes().size() > 1) { + throw new IllegalArgumentException(formatWithLocale( + "The supplied graph has more than one relationship type: %s", + StringJoining.join(relationshipSchema.availableTypes().stream().map(e -> e.name)) + )); + } + var nodeProperties = constructNodePropertiesFromGraph(graph); - var relationshipProperties = constructRelationshipPropertiesFromGraph( - graph, - relationshipType, - relationshipPropertyKey, - graph.relationshipProperties() - ); + RelationshipImportResult relationshipImportResult; + if (relationshipSchema.availableTypes().isEmpty()) { + relationshipImportResult = RelationshipImportResult.builder().build(); + } else { + var relationshipType = relationshipSchema.availableTypes().iterator().next(); - var relationshipImportResult = RelationshipImportResult.builder().putImportResult( - relationshipType, - SingleTypeRelationships.builder() - .topology(graph.relationshipTopology()) - .properties(relationshipProperties) - .direction(direction) - .build() - ).build(); + var relationshipProperties = constructRelationshipPropertiesFromGraph( + graph, + relationshipType, + relationshipPropertyKey, + graph.relationshipProperties() + ); - var schema = GraphSchema.of(NodeSchema.from(graph.schema().nodeSchema()), relationshipSchema, Map.of()); + relationshipImportResult = RelationshipImportResult.builder().putImportResult( + relationshipType, + SingleTypeRelationships.builder() + .relationshipSchemaEntry(relationshipSchema.get(relationshipType)) + .topology(graph.relationshipTopology()) + .properties(relationshipProperties) + .build() + ).build(); + } return new GraphStoreBuilder() .databaseId(databaseId) // TODO: is it correct that we only use this for generated graphs? .capabilities(ImmutableStaticCapabilities.of(false)) .schema(schema) - .nodes(Nodes.of(graph.idMap(), nodeProperties)) + .nodes(ImmutableNodes.of(schema.nodeSchema(), graph.idMap(), nodeProperties)) .relationshipImportResult(relationshipImportResult) .graphProperties(GraphPropertyStore.empty()) .concurrency(concurrency) @@ -182,70 +175,5 @@ private static Optional constructRelationshipProperti } - public static void extractNodeProperties( - ImmutableNodes.Builder nodeImportResultBuilder, - Function nodeSchema, - Map nodeProperties - ) { - NodePropertyStore.Builder propertyStoreBuilder = NodePropertyStore.builder(); - nodeProperties.forEach((propertyKey, propertyValues) -> { - var propertySchema = nodeSchema.apply(propertyKey); - propertyStoreBuilder.putIfAbsent( - propertyKey, - NodeProperty.of( - propertyKey, - propertySchema.state(), - propertyValues, - propertySchema.defaultValue() - ) - ); - }); - nodeImportResultBuilder.properties(propertyStoreBuilder.build()); - } - - public static GraphSchema computeGraphSchema( - Nodes nodes, - Function> propertiesByLabel, - RelationshipImportResult relationshipImportResult - ) { - var nodeProperties = nodes.properties().properties(); - - var nodeSchema = NodeSchema.empty(); - for (var label : nodes.idMap().availableNodeLabels()) { - var entry = nodeSchema.getOrCreateLabel(label); - for (var propertyKey : propertiesByLabel.apply(label)) { - entry.addProperty( - propertyKey, - nodeProperties.get(propertyKey).propertySchema() - ); - } - } - nodes.idMap().availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); - - var relationshipSchema = RelationshipSchema.empty(); - - relationshipImportResult.importResults().forEach(((relationshipType, singleTypeRelationshipImportResult) -> { - relationshipSchema.getOrCreateRelationshipType( - relationshipType, - singleTypeRelationshipImportResult.direction() - ); - singleTypeRelationshipImportResult.properties() - .map(RelationshipPropertyStore::relationshipProperties) - .ifPresent(properties -> properties.forEach((propertyKey, propertyValues) -> relationshipSchema - .getOrCreateRelationshipType( - relationshipType, - singleTypeRelationshipImportResult.direction() - ) - .addProperty(propertyKey, propertyValues.propertySchema()))); - })); - - return GraphSchema.of( - nodeSchema, - relationshipSchema, - Map.of() - ); - } - - private CSRGraphStoreUtil() {} } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index 700481b2dcf..9e9199fca9b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.core.loading; -import org.immutables.value.Value; import org.neo4j.gds.ElementProjection; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; @@ -28,66 +27,111 @@ import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.CSRGraphStoreFactory; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphLoaderContext; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.compat.GraphDatabaseApiProxy; -import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.GraphDimensionsCypherReader; import org.neo4j.gds.core.ImmutableGraphDimensions; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.TaskTreeProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory; import org.neo4j.gds.transaction.TransactionContext; -import org.neo4j.internal.id.IdGeneratorFactory; -import java.util.ArrayList; import java.util.Collection; -import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.LongStream; -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; +import static org.neo4j.gds.core.loading.CypherQueryEstimator.EstimationResult; import static org.neo4j.internal.kernel.api.security.AccessMode.Static.READ; -import static org.neo4j.kernel.api.StatementConstants.NO_SUCH_PROPERTY_KEY; -public class CypherFactory extends CSRGraphStoreFactory { +public final class CypherFactory extends CSRGraphStoreFactory { private final GraphProjectFromCypherConfig cypherConfig; - private EstimationResult nodeEstimation; - private EstimationResult relationshipEstimation; + private final long numberOfNodeProperties; + private final long numberOfRelationshipProperties; + private final ProgressTracker progressTracker; - public CypherFactory( + public static CypherFactory createWithBaseDimensions(GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext, GraphDimensions graphDimensions) { + return create(graphProjectConfig, loadingContext, Optional.of(graphDimensions)); + } + + public static CypherFactory createWithDerivedDimensions(GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext) { + return create(graphProjectConfig, loadingContext, Optional.empty()); + } + + private static CypherFactory create( GraphProjectFromCypherConfig graphProjectConfig, - GraphLoaderContext loadingContext + GraphLoaderContext loadingContext, + Optional baseDimensions ) { - this( + + EstimationResult nodeEstimation; + EstimationResult relationEstimation; + + if (graphProjectConfig.isFictitiousLoading()) { + nodeEstimation = ImmutableEstimationResult.of(graphProjectConfig.nodeCount(), 0); + relationEstimation = ImmutableEstimationResult.of(graphProjectConfig.relationshipCount(), 0); + } else { + var estimator = new CypherQueryEstimator(loadingContext.transactionContext().withRestrictedAccess(READ)); + nodeEstimation = estimator.getNodeEstimation(graphProjectConfig.nodeQuery()); + relationEstimation = estimator.getRelationshipEstimation(graphProjectConfig.relationshipQuery()); + } + + var dimBuilder = ImmutableGraphDimensions.builder(); + + baseDimensions.ifPresent(dimBuilder::from); + + long highestPossibleNodeCount = Math.max(baseDimensions + .map(GraphDimensions::highestPossibleNodeCount) + .orElse(-1L), nodeEstimation.estimatedRows()); + long nodeCount = Math.max( + baseDimensions.map(GraphDimensions::nodeCount).orElse(-1L), + nodeEstimation.estimatedRows() + ); + long relCountUpperBound = Math.max( + baseDimensions.map(GraphDimensions::relCountUpperBound).orElse(-1L), + relationEstimation.estimatedRows() + ); + + GraphDimensions dim = dimBuilder + .highestPossibleNodeCount(highestPossibleNodeCount) + .nodeCount(nodeCount) + .relCountUpperBound(relCountUpperBound) + .build(); + + return new CypherFactory( graphProjectConfig, loadingContext, - new GraphDimensionsCypherReader( - loadingContext.transactionContext().withRestrictedAccess(READ), - graphProjectConfig, - GraphDatabaseApiProxy.resolveDependency(loadingContext.graphDatabaseService(), IdGeneratorFactory.class) - ).call() + dim, + nodeEstimation.propertyCount(), + relationEstimation.propertyCount() ); } - public CypherFactory( + private CypherFactory( GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext, - GraphDimensions graphDimensions + GraphDimensions graphDimensions, + long estimatedNumberOfNodeProperties, + long estimatedNumberOfRelProperties ) { // TODO: need to pass capabilities from outside? super(graphProjectConfig, ImmutableStaticCapabilities.of(true), loadingContext, graphDimensions); - this.cypherConfig = getCypherConfig(graphProjectConfig).orElseThrow(() -> new IllegalArgumentException( - "Expected GraphProjectConfig to be a cypher config.")); + + this.cypherConfig = graphProjectConfig; + this.numberOfNodeProperties = estimatedNumberOfNodeProperties; + this.numberOfRelationshipProperties = estimatedNumberOfRelProperties; + this.progressTracker = initProgressTracker(); + } + + @Override + protected ProgressTracker progressTracker() { + return progressTracker; } @Override @@ -110,23 +154,7 @@ public MemoryEstimation estimateMemoryUsageAfterLoading() { @Override public GraphDimensions estimationDimensions() { - return ImmutableGraphDimensions.builder() - .from(dimensions) - .highestPossibleNodeCount(getNodeEstimation().estimatedRows()) - .nodeCount(getNodeEstimation().estimatedRows()) - .relCountUpperBound(getRelationshipEstimation().estimatedRows()) - .build(); - } - - @Override - protected GraphSchema computeGraphSchema( - Nodes nodes, RelationshipImportResult relationshipImportResult - ) { - return CSRGraphStoreUtil.computeGraphSchema( - nodes, - (__) -> nodes.properties().keySet(), - relationshipImportResult - ); + return dimensions; } @Override @@ -134,32 +162,32 @@ public CSRGraphStore build() { // Temporarily override the security context to enforce read-only access during load return readOnlyTransaction().apply((tx, ktx) -> { BatchLoadResult nodeCount = new CountingCypherRecordLoader( - nodeQuery(), + cypherConfig.nodeQuery(), CypherRecordLoader.QueryType.NODE, cypherConfig, loadingContext ).load(ktx.internalTransaction()); progressTracker.beginSubTask("Loading"); - var idMapAndProperties = new CypherNodeLoader( - nodeQuery(), + var nodes = new CypherNodeLoader( + cypherConfig.nodeQuery(), nodeCount.rows(), cypherConfig, loadingContext, progressTracker ).load(ktx.internalTransaction()); - var relationshipsAndProperties = new CypherRelationshipLoader( - relationshipQuery(), - idMapAndProperties.idMap(), + var relationshipImportResult = new CypherRelationshipLoader( + cypherConfig.relationshipQuery(), + nodes.idMap(), cypherConfig, loadingContext, progressTracker ).load(ktx.internalTransaction()); var graphStore = createGraphStore( - idMapAndProperties, - relationshipsAndProperties + nodes, + relationshipImportResult ); progressTracker.endSubTask("Loading"); @@ -170,14 +198,25 @@ public CSRGraphStore build() { }); } - @Override - protected ProgressTracker initProgressTracker() { + private ProgressTracker initProgressTracker() { var task = Tasks.task( "Loading", - Tasks.leaf("Nodes"), + Tasks.leaf("Nodes", dimensions.highestPossibleNodeCount()), Tasks.leaf("Relationships", dimensions.relCountUpperBound()) ); - return new TaskProgressTracker( + + if (graphProjectConfig.logProgress()) { + return new TaskProgressTracker( + task, + loadingContext.log(), + graphProjectConfig.readConcurrency(), + graphProjectConfig.jobId(), + loadingContext.taskRegistryFactory(), + EmptyUserLogRegistryFactory.INSTANCE + ); + } + + return new TaskTreeProgressTracker( task, loadingContext.log(), graphProjectConfig.readConcurrency(), @@ -187,72 +226,15 @@ protected ProgressTracker initProgressTracker() { ); } - private String nodeQuery() { - return getCypherConfig(graphProjectConfig) - .orElseThrow(() -> new IllegalArgumentException("Missing node query")) - .nodeQuery(); - } - - private String relationshipQuery() { - return getCypherConfig(graphProjectConfig) - .orElseThrow(() -> new IllegalArgumentException("Missing relationship query")) - .relationshipQuery(); - } - - private static Optional getCypherConfig(GraphProjectConfig config) { - if (config instanceof GraphProjectFromCypherConfig) { - return Optional.of((GraphProjectFromCypherConfig) config); - } - return Optional.empty(); - } - private TransactionContext readOnlyTransaction() { return loadingContext.transactionContext().withRestrictedAccess(READ); } - private EstimationResult getNodeEstimation() { - if (nodeEstimation == null) { - nodeEstimation = runEstimationQuery( - nodeQuery(), - NodeSubscriber.RESERVED_COLUMNS - ); - } - return nodeEstimation; - } - - private EstimationResult getRelationshipEstimation() { - if (relationshipEstimation == null) { - relationshipEstimation = runEstimationQuery( - relationshipQuery(), - RelationshipSubscriber.RESERVED_COLUMNS - ); - } - return relationshipEstimation; - } - - private EstimationResult runEstimationQuery(String query, Collection reservedColumns) { - return readOnlyTransaction().apply((tx, ktx) -> { - var explainQuery = formatWithLocale("EXPLAIN %s", query); - try (var result = tx.execute(explainQuery)) { - var estimatedRows = (Number) result.getExecutionPlanDescription().getArguments().get("EstimatedRows"); - - var propertyColumns = new ArrayList<>(result.columns()); - propertyColumns.removeAll(reservedColumns); - - return ImmutableEstimationResult.of(estimatedRows.longValue(), propertyColumns.size()); - } - }); - } - private NodeProjections buildEstimateNodeProjections() { - if (cypherConfig.isFictitiousLoading()) { - nodeEstimation = ImmutableEstimationResult.of(cypherConfig.nodeCount(), 0); - } - var nodeProjection = NodeProjection .builder() .label(ElementProjection.PROJECT_ALL) - .addAllProperties(getNodeEstimation().propertyMappings()) + .addAllProperties(propertyMappings(numberOfNodeProperties)) .build(); return NodeProjections.single( @@ -262,14 +244,10 @@ private NodeProjections buildEstimateNodeProjections() { } private RelationshipProjections buildEstimateRelationshipProjections() { - if (cypherConfig.isFictitiousLoading()) { - relationshipEstimation = ImmutableEstimationResult.of(cypherConfig.relationshipCount(), 0); - } - var relationshipProjection = RelationshipProjection .builder() .type(ElementProjection.PROJECT_ALL) - .addAllProperties(getRelationshipEstimation().propertyMappings()) + .addAllProperties(propertyMappings(numberOfRelationshipProperties)) .build(); return RelationshipProjections.single( @@ -278,30 +256,10 @@ private RelationshipProjections buildEstimateRelationshipProjections() { ); } - @ValueClass - interface EstimationResult { - long estimatedRows(); - - long propertyCount(); - - @Value.Derived - default Map propertyTokens() { - return LongStream - .range(0, propertyCount()) - .boxed() - .collect(Collectors.toMap( - Object::toString, - property -> NO_SUCH_PROPERTY_KEY - )); - } - - @Value.Derived - default Collection propertyMappings() { - return LongStream - .range(0, propertyCount()) - .mapToObj(property -> PropertyMapping.of(Long.toString(property), DefaultValue.DEFAULT)) - .collect(Collectors.toList()); - } - + private static Collection propertyMappings(long propertyCount) { + return LongStream + .range(0, propertyCount) + .mapToObj(property -> PropertyMapping.of(Long.toString(property), DefaultValue.DEFAULT)) + .collect(Collectors.toList()); } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java index 1614190ff96..f6b85b0a2ca 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java @@ -20,19 +20,15 @@ package org.neo4j.gds.core.loading; import org.immutables.value.Value; -import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodesBuilder; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.kernel.impl.coreapi.InternalTransaction; -import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; @Value.Enclosing class CypherNodeLoader extends CypherRecordLoader { @@ -71,6 +67,7 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { .maxOriginalId(NodesBuilder.UNKNOWN_MAX_ID) .hasLabelInformation(hasLabelInformation) .hasProperties(!propertyColumns.isEmpty()) + .propertyState(PropertyState.TRANSIENT) .build(); nodeSubscriber.initialize(subscription.fieldNames(), this.nodesBuilder); @@ -99,12 +96,7 @@ void updateCounts(BatchLoadResult result) { @Override Nodes result() { - var nodes = nodesBuilder.build(highestNodeId); - var idMap = nodes.idMap(); - var nodeProperties = nodes.properties().propertyValues(); - var nodePropertiesWithPropertyMappings = propertiesWithPropertyMappings(nodeProperties); - - return Nodes.of(idMap, nodePropertiesWithPropertyMappings, PropertyState.TRANSIENT); + return nodesBuilder.build(highestNodeId); } @Override @@ -121,16 +113,4 @@ Set getReservedColumns() { QueryType queryType() { return QueryType.NODE; } - - private static Map propertiesWithPropertyMappings(Map properties) { - return properties.entrySet() - .stream() - .collect(Collectors.toMap( - propertiesByKey -> PropertyMapping.of( - propertiesByKey.getKey(), - propertiesByKey.getValue().valueType().fallbackValue() - ), - Map.Entry::getValue - )); - } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java new file mode 100644 index 00000000000..8de1954c8de --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.transaction.TransactionContext; + +import java.util.ArrayList; +import java.util.Collection; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; +import static org.neo4j.internal.kernel.api.security.AccessMode.Static.READ; + +public class CypherQueryEstimator { + + private final TransactionContext context; + + CypherQueryEstimator(TransactionContext context) {this.context = context;} + + + EstimationResult getNodeEstimation(String nodeQuery) { + return runEstimationQuery(nodeQuery, NodeSubscriber.RESERVED_COLUMNS); + } + + EstimationResult getRelationshipEstimation(String relationshipQuery) { + return runEstimationQuery( + relationshipQuery, + RelationshipSubscriber.RESERVED_COLUMNS + ); + } + + private EstimationResult runEstimationQuery(String query, Collection reservedColumns) { + return context.withRestrictedAccess(READ).apply((tx, ktx) -> { + var explainQuery = formatWithLocale("EXPLAIN %s", query); + try (var result = tx.execute(explainQuery)) { + var estimatedRows = (Number) result.getExecutionPlanDescription().getArguments().get("EstimatedRows"); + + var propertyColumns = new ArrayList<>(result.columns()); + propertyColumns.removeAll(reservedColumns); + + return ImmutableEstimationResult.of(estimatedRows.longValue(), propertyColumns.size()); + } + }); + } + + @ValueClass + public + interface EstimationResult { + long estimatedRows(); + + long propertyCount(); + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java index 9f64d8a2282..636bbd9a7e3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java @@ -21,24 +21,27 @@ import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.utils.StringJoining; import org.neo4j.graphdb.security.AuthorizationViolationException; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.query.QueryExecution; import org.neo4j.kernel.impl.query.QueryExecutionEngine; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.util.ValueUtils; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.neo4j.gds.compat.GraphDatabaseApiProxy.runQueryWithoutClosingTheResult; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; abstract class CypherRecordLoader { @@ -146,4 +149,22 @@ private void validateMandatoryColumns(Collection allColumns) { )); } } + + private static QueryExecution runQueryWithoutClosingTheResult( + InternalTransaction tx, + String query, + Map params, + TransactionalContextFactory contextFactory, + QueryExecutionEngine executionEngine, + QuerySubscriber subscriber + ) { + var convertedParams = ValueUtils.asMapValue(params); + var context = Neo4jProxy.newQueryContext(contextFactory, tx, query, convertedParams); + try { + return executionEngine.executeQuery(query, convertedParams, context, false, subscriber); + } catch (QueryExecutionKernelException e) { + throw e.asUserException(); + } + } + } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java index 4d6c3023807..fd081f80ea5 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java @@ -19,14 +19,12 @@ */ package org.neo4j.gds.core.loading; -import org.apache.commons.lang3.mutable.MutableInt; import org.eclipse.collections.impl.map.mutable.primitive.ObjectDoubleHashMap; -import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap; import org.immutables.value.Value; +import org.neo4j.gds.ImmutablePropertyMappings; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; -import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphLoaderContext; @@ -51,14 +49,9 @@ class CypherRelationshipLoader extends CypherRecordLoader propertyKeyIdsByName; private ObjectDoubleHashMap propertyDefaultValueByName; private boolean initializedFromResult; private List propertyConfigs; - private RelationshipProjection.Builder projectionBuilder; CypherRelationshipLoader( String relationshipQuery, @@ -74,15 +67,7 @@ class CypherRelationshipLoader extends CypherRecordLoader(numberOfMappings); - propertyMappings - .stream() - .forEach(mapping -> propertyKeyIdsByName.put(mapping.neoPropertyKey(), propertyKeyId.getAndIncrement())); - - propertyDefaultValueByName = new ObjectDoubleHashMap<>(numberOfMappings); + propertyDefaultValueByName = new ObjectDoubleHashMap<>(propertyMappings.numberOfMappings()); propertyMappings .stream() .forEach(mapping -> propertyDefaultValueByName.put( @@ -98,11 +83,6 @@ private void initFromPropertyMappings(PropertyMappings propertyMappings) { mapping.defaultValue() )) .collect(Collectors.toList()); - - projectionBuilder = RelationshipProjection - .builder() - .orientation(Orientation.NATURAL) - .properties(propertyMappings); } @Override @@ -123,12 +103,15 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { )) .collect(Collectors.toList()); - initFromPropertyMappings(PropertyMappings.of(propertyMappings)); + initFromPropertyMappings(ImmutablePropertyMappings.of(propertyMappings)); initializedFromResult = true; } subscriber.initialize(subscription.fieldNames(), propertyDefaultValueByName); CypherLoadingUtils.consume(subscription); + subscriber.error().ifPresent(e -> { + throw e; + }); progressTracker.endSubTask("Relationships"); return new BatchLoadResult(subscriber.rows(), -1L); } @@ -175,6 +158,7 @@ RelationshipsBuilder getOrCreateRelationshipsBuilder(RelationshipType relationsh private RelationshipsBuilder createRelationshipsBuilder(RelationshipType relationshipType) { return GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(relationshipType) .concurrency(cypherConfig.readConcurrency()) .propertyConfigs(propertyConfigs) .orientation(Orientation.NATURAL) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java index 306a55710de..e9c36073b79 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java @@ -24,12 +24,18 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.utils.ExceptionUtil; import org.neo4j.gds.utils.StringJoining; +import org.neo4j.logging.Log; +import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -41,8 +47,26 @@ public final class GraphStoreCatalog { private static final ConcurrentHashMap userCatalogs = new ConcurrentHashMap<>(); + private static final Set listeners = new HashSet<>(); + + // as we want to use the Neo4j log if possible and the catalog is a static instance, + // we make the log injectable + private static Optional log = Optional.empty(); + private GraphStoreCatalog() { } + public static void registerListener(GraphStoreCatalogListener listener) { + listeners.add(listener); + } + + public static void unregisterListener(GraphStoreCatalogListener listener) { + listeners.remove(listener); + } + + public static void setLog(Log log) { + GraphStoreCatalog.log = Optional.of(log); + } + public static GraphStoreWithConfig get(CatalogRequest request, String graphName) { var userCatalogKey = UserCatalog.UserCatalogKey.of(request.databaseName(), graphName); var ownCatalog = getUserCatalog(request.username()); @@ -161,6 +185,22 @@ private static void set(GraphProjectConfig config, GraphStore graphStore, boolea ); return userCatalog; }); + + listeners.forEach(listener -> ExceptionUtil.safeRunWithLogException( + log.orElseGet(Neo4jProxy::testLog), + () -> String.format( + Locale.US, + "Could not call listener %s on setting the graph %s", + listener, + config.graphName() + ), + () -> listener.onProject( + config.username(), + graphStore.databaseId().databaseName(), + config.graphName() + ) + ) + ); } public static boolean exists(String username, String databaseName, String graphName) { diff --git a/core/src/main/java/org/neo4j/gds/config/ComponentSizeConfig.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java similarity index 77% rename from core/src/main/java/org/neo4j/gds/config/ComponentSizeConfig.java rename to core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java index 622ded466c6..3b28be35ab0 100644 --- a/core/src/main/java/org/neo4j/gds/config/ComponentSizeConfig.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java @@ -17,14 +17,9 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.gds.config; +package org.neo4j.gds.core.loading; -import org.neo4j.gds.annotation.Configuration; +public interface GraphStoreCatalogListener { -import java.util.Optional; - -public interface ComponentSizeConfig { - - @Configuration.LongRange(min = 1L) - Optional minComponentSize(); + void onProject(String user, String database, String graphName); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java b/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java index 806bc3fe778..54cc9001789 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java @@ -46,24 +46,29 @@ import static java.util.stream.Collectors.toMap; import static org.neo4j.gds.core.GraphDimensions.ANY_LABEL; +// TODO: should be named LoadablePropertyMappings final class IndexPropertyMappings { - static LoadablePropertyMappings prepareProperties( - GraphProjectFromStoreConfig graphProjectConfig, - GraphDimensions graphDimensions, - TransactionContext transaction - ) { - Map storeLoadedProperties = graphProjectConfig + static Map propertyMappings(GraphProjectFromStoreConfig graphProjectConfig) { + return graphProjectConfig .nodeProjections() .projections() .entrySet() .stream() .collect(toMap( Map.Entry::getKey, - entry -> entry.getValue().properties() + entry -> entry + .getValue() + .properties() )); + } - return prepareLoadableProperties(graphDimensions, transaction, storeLoadedProperties); + static LoadablePropertyMappings prepareProperties( + GraphProjectFromStoreConfig graphProjectConfig, + GraphDimensions graphDimensions, + TransactionContext transaction + ) { + return prepareLoadableProperties(graphDimensions, transaction, propertyMappings(graphProjectConfig)); } private static LoadablePropertyMappings prepareLoadableProperties( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index e05fea8ba7a..f0c5597e1f6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -20,17 +20,16 @@ package org.neo4j.gds.core.loading; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PartialIdMap; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.properties.nodes.NodePropertyStore; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodeLabelToken; import org.neo4j.gds.core.loading.construction.NodesBuilder; import org.neo4j.gds.core.loading.construction.PropertyValues; import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; -import java.util.Map; -import java.util.Optional; +import java.util.Locale; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicBoolean; @@ -56,33 +55,47 @@ public void prepareForFlush() { } public long addNode(long nodeId, NodeLabelToken nodeLabels) { - var intermediateId = this.intermediateIdMapBuilder.toMappedNodeId(nodeId); + checkPositiveId(nodeId); + + long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); // deduplication - if (intermediateId != IdMap.NOT_FOUND) { - return intermediateId; + if (intermediateId < 0) { + return -(intermediateId + 1); } - intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); - this.nodesBuilder.addNode(intermediateId, nodeLabels); return intermediateId; } + /** + * GDS has the general assumption of non-negative original node ids. + */ + private static void checkPositiveId(long nodeId) { + if (nodeId < 0) { + throw new IllegalArgumentException(String.format( + Locale.US, + "GDS expects node ids to be positive. But got a negative id of `%d`.", + nodeId + )); + } + } + public long addNodeWithProperties( long nodeId, PropertyValues properties, NodeLabelToken nodeLabels ) { - var intermediateId = this.intermediateIdMapBuilder.toMappedNodeId(nodeId); + long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); + + checkPositiveId(nodeId); // deduplication - if (intermediateId != IdMap.NOT_FOUND) { - return intermediateId; + if (intermediateId < 0) { + return -(intermediateId + 1); } - intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); if (properties.isEmpty()) { this.nodesBuilder.addNode(intermediateId, nodeLabels); } else { @@ -110,7 +123,9 @@ public interface HighLimitIdMapAndProperties { PartialIdMap intermediateIdMap(); - Optional> nodeProperties(); + MutableNodeSchema schema(); + + NodePropertyStore propertyStore(); } public HighLimitIdMapAndProperties build() { @@ -141,7 +156,8 @@ public OptionalLong rootNodeCount() { .builder() .idMap(idMap) .intermediateIdMap(partialIdMap) - .nodeProperties(nodes.properties().propertyValues()) + .schema(nodes.schema()) + .propertyStore(nodes.properties()) .build(); } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java index bba0021e742..e6cfedaf7ac 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java @@ -28,7 +28,6 @@ import org.neo4j.gds.api.CSRGraphStoreFactory; import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.GraphDimensions; @@ -44,6 +43,7 @@ import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.TaskTreeProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory; import org.neo4j.internal.id.IdGeneratorFactory; @@ -56,6 +56,7 @@ public final class NativeFactory extends CSRGraphStoreFactory { private final GraphProjectFromStoreConfig storeConfig; + private final ProgressTracker progressTracker; public NativeFactory( GraphProjectFromStoreConfig graphProjectConfig, @@ -79,6 +80,7 @@ public NativeFactory( ) { super(graphProjectConfig, ImmutableStaticCapabilities.of(true), loadingContext, graphDimensions); this.storeConfig = graphProjectConfig; + this.progressTracker = initProgressTracker(); } @Override @@ -250,8 +252,7 @@ private static void relationshipEstimationAfterLoading( }); } - @Override - protected ProgressTracker initProgressTracker() { + private ProgressTracker initProgressTracker() { long relationshipCount = graphProjectConfig .relationshipProjections() .projections() @@ -285,7 +286,19 @@ protected ProgressTracker initProgressTracker() { Tasks.task("Nodes", nodeTasks), Tasks.task("Relationships", Tasks.leaf("Store Scan", relationshipCount)) ); - return new TaskProgressTracker( + + if (graphProjectConfig.logProgress()) { + return new TaskProgressTracker( + task, + loadingContext.log(), + graphProjectConfig.readConcurrency(), + graphProjectConfig.jobId(), + loadingContext.taskRegistryFactory(), + EmptyUserLogRegistryFactory.INSTANCE + ); + } + + return new TaskTreeProgressTracker( task, loadingContext.log(), graphProjectConfig.readConcurrency(), @@ -295,17 +308,6 @@ protected ProgressTracker initProgressTracker() { ); } - @Override - protected GraphSchema computeGraphSchema( - Nodes nodes, RelationshipImportResult relationshipImportResult - ) { - return CSRGraphStoreUtil.computeGraphSchema( - nodes, - (label) -> storeConfig.nodeProjections().projections().get(label).properties().propertyKeys(), - relationshipImportResult - ); - } - @Override public CSRGraphStore build() { validate(dimensions, storeConfig); @@ -359,4 +361,9 @@ private RelationshipImportResult loadRelationships(IdMap idMap, int concurrency) progressTracker.endSubTask(); } } + + @Override + protected ProgressTracker progressTracker() { + return progressTracker; + } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NodeImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/NodeImporter.java index de38ea8fbe1..729f9ace438 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NodeImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NodeImporter.java @@ -20,12 +20,14 @@ package org.neo4j.gds.core.loading; import com.carrotsearch.hppc.IntObjectMap; +import org.immutables.builder.Builder; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.compat.PropertyReference; import org.neo4j.gds.core.utils.RawValues; import java.util.Collections; import java.util.List; +import java.util.Optional; public class NodeImporter { @@ -35,13 +37,14 @@ public interface PropertyReader { private final IdMapBuilder idMapBuilder; private final LabelInformation.Builder labelInformationBuilder; - private final IntObjectMap> labelTokenNodeLabelMapping; + private final Optional>> labelTokenNodeLabelMapping; private final boolean importProperties; - public NodeImporter( + @Builder.Constructor + NodeImporter( IdMapBuilder idMapBuilder, LabelInformation.Builder labelInformationBuilder, - IntObjectMap> labelTokenNodeLabelMapping, + Optional>> labelTokenNodeLabelMapping, boolean importProperties ) { this.idMapBuilder = idMapBuilder; @@ -51,6 +54,13 @@ public NodeImporter( } public long importNodes(NodesBatchBuffer buffer, PropertyReader reader) { + var tokenToLabelMap = this.labelTokenNodeLabelMapping.orElseThrow( + () -> new IllegalStateException("Missing Token-to-NodeLabel mapping") + ); + return importNodes(buffer, tokenToLabelMap, reader); + } + + public long importNodes(NodesBatchBuffer buffer, IntObjectMap> tokenToNodeLabelsMap, PropertyReader reader) { int batchLength = buffer.length(); if (batchLength == 0) { return 0; @@ -84,7 +94,8 @@ public long importNodes(NodesBatchBuffer buffer, PropertyReader reader) { batch, batchLength, labelIds, - (nodeIds, pos) -> nodeIds[pos] + (nodeIds, pos) -> nodeIds[pos], + tokenToNodeLabelsMap ); } @@ -96,19 +107,22 @@ public long importNodes(NodesBatchBuffer buffer, PropertyReader reader) { return RawValues.combineIntInt(batchLength, importedProperties); } - private void setNodeLabelInformation(long[] batch, int batchLength, long[][] labelIds, IdFunction idFunction) { + private void setNodeLabelInformation( + long[] batch, + int batchLength, + long[][] labelIds, + IdFunction idFunction, + IntObjectMap> tokenToNodeLabelsMap + ) { int cappedBatchLength = Math.min(labelIds.length, batchLength); for (int i = 0; i < cappedBatchLength; i++) { long nodeId = idFunction.apply(batch, i); - long[] labelIdsForNode = labelIds[i]; - - for (long labelId : labelIdsForNode) { - var elementIdentifiers = labelTokenNodeLabelMapping.getOrDefault( - (int) labelId, - Collections.emptyList() - ); - for (NodeLabel elementIdentifier : elementIdentifiers) { - labelInformationBuilder.addNodeIdToLabel(elementIdentifier, nodeId); + long[] labelTokensForNode = labelIds[i]; + + for (long token : labelTokensForNode) { + var nodeLabels = tokenToNodeLabelsMap.getOrDefault((int) token, Collections.emptyList()); + for (NodeLabel nodeLabel : nodeLabels) { + this.labelInformationBuilder.addNodeIdToLabel(nodeLabel, nodeId); } } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 33695c63970..c35d9929221 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -20,19 +20,25 @@ package org.neo4j.gds.core.loading; import org.immutables.value.Value; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.PropertyMapping; +import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.properties.nodes.NodeProperty; +import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.schema.ImmutablePropertySchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import java.util.Map; @ValueClass public interface Nodes { + MutableNodeSchema schema(); + IdMap idMap(); @Value.Default @@ -40,27 +46,42 @@ default NodePropertyStore properties() { return NodePropertyStore.empty(); } - static Nodes of(IdMap idmap) { - return ImmutableNodes.of(idmap, NodePropertyStore.empty()); - } + static Nodes of( + IdMap idMap, + Map propertyMappings, + Map propertyValues, + PropertyState propertyState + ) { + var nodeSchema = MutableNodeSchema.empty(); + var nodePropertyStoreBuilder = NodePropertyStore.builder(); - static Nodes of(IdMap idmap, NodePropertyStore nodePropertyStore) { - return ImmutableNodes.of(idmap, nodePropertyStore); - } + propertyMappings.forEach(((nodeLabel, mappings) -> { + if (mappings.mappings().isEmpty()) { + nodeSchema.addLabel(nodeLabel); + } else { + mappings.mappings().forEach(propertyMapping -> { + var nodePropertyValues = propertyValues.get(propertyMapping); + // The default value is either overridden by the user + // or inferred from the actual property value. + var defaultValue = propertyMapping.defaultValue().isUserDefined() + ? propertyMapping.defaultValue() + : nodePropertyValues.valueType().fallbackValue(); + var propertySchema = ImmutablePropertySchema.builder() + .key(propertyMapping.propertyKey()) + .valueType(nodePropertyValues.valueType()) + .defaultValue(defaultValue) + .state(propertyState) + .build(); + + nodeSchema.addProperty(nodeLabel, propertySchema.key(), propertySchema); + nodePropertyStoreBuilder.putProperty( + propertySchema.key(), + ImmutableNodeProperty.of(nodePropertyValues, propertySchema) + ); + }); + } + })); - static Nodes of(IdMap idMap, Map properties, PropertyState propertyState) { - NodePropertyStore.Builder builder = NodePropertyStore.builder(); - properties.forEach((mapping, nodeProperties) -> builder.putProperty( - mapping.propertyKey(), - NodeProperty.of( - mapping.propertyKey(), - propertyState, - nodeProperties, - mapping.defaultValue().isUserDefined() - ? mapping.defaultValue() - : nodeProperties.valueType().fallbackValue() - ) - )); - return ImmutableNodes.of(idMap, builder.build()); + return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NullPropertyMap.java b/core/src/main/java/org/neo4j/gds/core/loading/NullPropertyMap.java index 2f7b0a8cbf3..dbf35141ef7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NullPropertyMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NullPropertyMap.java @@ -65,7 +65,7 @@ public OptionalDouble getMaxDoublePropertyValue() { } @Override - public long size() { + public long nodeCount() { return 0; } } @@ -101,7 +101,7 @@ public ValueType valueType() { } @Override - public long size() { + public long nodeCount() { return 0; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java index 354f95b9ff3..6343b748007 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.loading; +import org.immutables.value.Value; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; @@ -32,6 +33,8 @@ import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.ValueTypes; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.values.storable.NumberType; import java.util.Collection; @@ -45,6 +48,15 @@ public interface RelationshipImportResult { Map importResults(); + @Value.Lazy + default MutableRelationshipSchema relationshipSchema() { + var relationshipSchema = MutableRelationshipSchema.empty(); + + importResults().forEach((__, relationships) -> relationshipSchema.set(relationships.relationshipSchemaEntry())); + + return relationshipSchema; + } + static ImmutableRelationshipImportResult.Builder builder() { return ImmutableRelationshipImportResult.builder(); } @@ -56,14 +68,19 @@ static RelationshipImportResult of( ) { var relationshipImportResultBuilder = RelationshipImportResult.builder(); - topologies.forEach((relationshipType, topology) -> relationshipImportResultBuilder.putImportResult( - relationshipType, - SingleTypeRelationships.builder() - .topology(topology) - .properties(Optional.ofNullable(properties.get(relationshipType))) - .direction(directions.get(relationshipType)) - .build() - )); + topologies.forEach((relationshipType, topology) -> { + Direction direction = directions.get(relationshipType); + var schemaEntry = new MutableRelationshipSchemaEntry(relationshipType, direction); + + + relationshipImportResultBuilder.putImportResult( + relationshipType, + SingleTypeRelationships.builder() + .topology(topology) + .properties(Optional.ofNullable(properties.get(relationshipType))) + .build() + ); + }); return relationshipImportResultBuilder.build(); } @@ -105,9 +122,20 @@ static RelationshipImportResult of(Collection props + .relationshipProperties() + .forEach((key, prop) -> schemaEntry.addProperty(key, prop.propertySchema()))); + var importResultBuilder = builders.computeIfAbsent( importContext.relationshipType(), - relationshipType -> SingleTypeRelationships.builder().direction(direction) + relationshipType -> SingleTypeRelationships + .builder() + .relationshipSchemaEntry(schemaEntry) ); if (isInverseRelationship) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java index 2658ad2810c..fbdffd32e1c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java @@ -25,11 +25,13 @@ import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; import org.neo4j.values.storable.NumberValue; import org.neo4j.values.storable.TextValue; +import java.util.Optional; import java.util.Set; import static org.neo4j.gds.RelationshipType.ALL_RELATIONSHIPS; @@ -65,6 +67,8 @@ class RelationshipSubscriber implements QuerySubscriber { private RelationshipsBuilder allRelationshipsBuilder; + private Optional error = Optional.empty(); + RelationshipSubscriber( IdMap idMap, CypherRelationshipLoader.Context loaderContext, @@ -105,6 +109,10 @@ void initialize(String[] fieldNames, ObjectDoubleMap propertyDefaultValu this.propertyValueBuffer = new double[propertyCount]; } + Optional error() { + return error; + } + public long rows() { return rows; } @@ -183,7 +191,13 @@ public void onRecordCompleted() { } @Override public void onError(Throwable throwable) { - throw new RuntimeException(throwable); + if (throwable instanceof RuntimeException) { + this.error = Optional.of((RuntimeException) throwable); + } else if (throwable instanceof QueryExecutionKernelException) { + this.error = Optional.of(((QueryExecutionKernelException) throwable).asUserException()); + } else { + this.error = Optional.of(new RuntimeException(throwable)); + } } @Override diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 6f8668c8298..93d3a3863d6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -21,7 +21,9 @@ import org.immutables.builder.Builder; import org.jetbrains.annotations.Nullable; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.PropertyMapping; +import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; @@ -50,6 +52,7 @@ public final class ScanningNodesImporter extends ScanningRecordsImporter { private final IndexPropertyMappings.LoadablePropertyMappings propertyMappings; + private final Map propertyMappingsByLabel; private final TerminationFlag terminationFlag; private final IdMapBuilder idMapBuilder; private final LabelInformation.Builder labelInformationBuilder; @@ -87,14 +90,16 @@ public static ScanningNodesImporter scanningNodesImporter( ); } - var propertyMappings = IndexPropertyMappings.prepareProperties( + var propertyMappings = IndexPropertyMappings.propertyMappings(graphProjectConfig); + + var loadablePropertyMappings = IndexPropertyMappings.prepareProperties( graphProjectConfig, dimensions, loadingContext.transactionContext() ); var nodePropertyImporter = initializeNodePropertyImporter( - propertyMappings, + loadablePropertyMappings, dimensions, concurrency ); @@ -106,6 +111,7 @@ public static ScanningNodesImporter scanningNodesImporter( progressTracker, concurrency, propertyMappings, + loadablePropertyMappings, nodePropertyImporter, idMapBuilder, labelInformationBuilder @@ -118,6 +124,7 @@ private ScanningNodesImporter( GraphDimensions dimensions, ProgressTracker progressTracker, int concurrency, + Map propertyMappingsByLabel, IndexPropertyMappings.LoadablePropertyMappings propertyMappings, @Nullable NativeNodePropertyImporter nodePropertyImporter, IdMapBuilder idMapBuilder, @@ -133,6 +140,7 @@ private ScanningNodesImporter( this.terminationFlag = loadingContext.terminationFlag(); this.propertyMappings = propertyMappings; + this.propertyMappingsByLabel = propertyMappingsByLabel; this.nodePropertyImporter = nodePropertyImporter; this.idMapBuilder = idMapBuilder; this.labelInformationBuilder = labelInformationBuilder; @@ -156,12 +164,12 @@ public RecordScannerTaskRunner.RecordScannerTaskFactory recordScannerTaskFactory ImportSizing sizing, StoreScanner storeScanner ) { - var nodeImporter = new NodeImporter( - idMapBuilder, - labelInformationBuilder, - dimensions.tokenNodeLabelMapping(), - nodePropertyImporter != null - ); + var nodeImporter = new NodeImporterBuilder() + .idMapBuilder(idMapBuilder) + .labelInformationBuilder(labelInformationBuilder) + .labelTokenNodeLabelMapping(dimensions.tokenNodeLabelMapping()) + .importProperties(nodePropertyImporter != null) + .build(); return NodesScannerTask.factory( transaction, @@ -191,7 +199,7 @@ public Nodes build() { importPropertiesFromIndex(idMap, nodeProperties); } - return Nodes.of(idMap, nodeProperties, PropertyState.PERSISTENT); + return Nodes.of(idMap, this.propertyMappingsByLabel, nodeProperties, PropertyState.PERSISTENT); } private void importPropertiesFromIndex( @@ -257,7 +265,7 @@ private void importPropertiesFromIndex( for (var entry : buildersByPropertyKey.entrySet()) { NodePropertyValues propertyValues = entry.getValue().build(idMap); nodeProperties.put(entry.getKey(), propertyValues); - recordsImported += propertyValues.size(); + recordsImported += propertyValues.nodeCount(); } long tookNanos = System.nanoTime() - indexStart; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java index 4c9a0ad6aec..e4a681e6f70 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java @@ -27,8 +27,8 @@ import org.neo4j.gds.api.RelationshipPropertyStore; import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import java.util.Optional; @@ -38,43 +38,20 @@ public interface SingleTypeRelationships { SingleTypeRelationships EMPTY = SingleTypeRelationships .builder() - .direction(Direction.DIRECTED) + .relationshipSchemaEntry(new MutableRelationshipSchemaEntry(RelationshipType.of("REL"), Direction.DIRECTED)) .topology(Topology.EMPTY) .build(); - // TODO: figure out if we can remove this. - Direction direction(); - Topology topology(); + MutableRelationshipSchemaEntry relationshipSchemaEntry(); + Optional properties(); Optional inverseTopology(); Optional inverseProperties(); - default RelationshipSchema relationshipSchema(RelationshipType relationshipType) { - var schema = RelationshipSchema.empty(); - this.updateRelationshipSchemaEntry(schema.getOrCreateRelationshipType(relationshipType, direction())); - return schema; - } - - default void updateRelationshipSchemaEntry(RelationshipSchemaEntry schemaEntry) { - properties().ifPresent(relationshipPropertyStore -> relationshipPropertyStore - .relationshipProperties() - .forEach((propertyKey, relationshipProperty) -> { - schemaEntry.addProperty( - propertyKey, - RelationshipPropertySchema.of(propertyKey, - relationshipProperty.valueType(), - relationshipProperty.defaultValue(), - relationshipProperty.propertyState(), - relationshipProperty.aggregation() - ) - ); - })); - } - /** * Filters the relationships to include only the given property if present. */ @@ -86,9 +63,14 @@ default SingleTypeRelationships filter(String propertyKey) { relationshipPropertyStore.filter(propertyKey) ); + + RelationshipSchemaEntry entry = relationshipSchemaEntry(); + var filteredEntry = new MutableRelationshipSchemaEntry(entry.identifier(), entry.direction()) + .addProperty(propertyKey, entry.properties().get(propertyKey)); + return SingleTypeRelationships.builder() .topology(topology()) - .direction(direction()) + .relationshipSchemaEntry(filteredEntry) .inverseTopology(inverseTopology()) .properties(properties) .inverseProperties(inverseProperties) @@ -111,14 +93,18 @@ static ImmutableSingleTypeRelationships.Builder builder() { } static SingleTypeRelationships of( + RelationshipType relationshipType, Topology topology, Direction direction, Optional properties, Optional propertySchema ) { + var schemaEntry = new MutableRelationshipSchemaEntry(relationshipType, direction); + propertySchema.ifPresent(schema -> schemaEntry.addProperty(schema.key(), schema)); + return SingleTypeRelationships.builder() - .direction(direction) .topology(topology) + .relationshipSchemaEntry(schemaEntry) .properties( propertySchema.map(schema -> { var relationshipProperty = ImmutableRelationshipProperty.builder() diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index d496997e5a9..10dd36417f2 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -19,13 +19,9 @@ */ package org.neo4j.gds.core.loading.construction; -import com.carrotsearch.hppc.IntObjectHashMap; -import com.carrotsearch.hppc.ObjectIntScatterMap; -import org.apache.commons.lang3.mutable.MutableInt; import org.immutables.builder.Builder; import org.immutables.value.Value; import org.neo4j.gds.ImmutableRelationshipProjection; -import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; @@ -38,6 +34,9 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; @@ -50,18 +49,14 @@ import org.neo4j.gds.core.loading.RecordsBatchBuffer; import org.neo4j.gds.core.loading.SingleTypeRelationshipImporterBuilder; import org.neo4j.gds.core.loading.SingleTypeRelationships; -import org.neo4j.gds.core.loading.nodeproperties.NodePropertiesFromStoreBuilder; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.stream.IntStream; -import static java.util.stream.Collectors.toMap; -import static org.neo4j.gds.core.GraphDimensions.ANY_LABEL; import static org.neo4j.kernel.api.StatementConstants.NO_SUCH_RELATIONSHIP_TYPE; @Value.Style( @@ -89,7 +84,8 @@ static NodesBuilder nodesBuilder( Optional hasLabelInformation, Optional hasProperties, Optional deduplicateIds, - Optional concurrency + Optional concurrency, + Optional propertyState ) { boolean labelInformation = nodeSchema .map(schema -> !(schema.availableLabels().isEmpty() && schema.containsOnlyAllNodesLabel())) @@ -119,13 +115,12 @@ static NodesBuilder nodesBuilder( )).orElseGet(() -> new NodesBuilder( maxOriginalId, threadCount, - new ObjectIntScatterMap<>(), - new IntObjectHashMap<>(), - new ConcurrentHashMap<>(), + NodesBuilderContext.lazy(threadCount), idMapBuilder, labelInformation, hasProperties.orElse(false), - deduplicate + deduplicate, + __ -> propertyState.orElse(PropertyState.PERSISTENT) )); } @@ -137,35 +132,15 @@ private static NodesBuilder fromSchema( boolean hasLabelInformation, boolean deduplicateIds ) { - var nodeLabels = nodeSchema.availableLabels(); - - var elementIdentifierLabelTokenMapping = new ObjectIntScatterMap(); - var labelTokenNodeLabelMapping = new IntObjectHashMap>(); - var labelTokenCounter = new MutableInt(0); - nodeLabels.forEach(nodeLabel -> { - int labelToken = nodeLabel == NodeLabel.ALL_NODES - ? ANY_LABEL - : labelTokenCounter.getAndIncrement(); - - elementIdentifierLabelTokenMapping.put(nodeLabel, labelToken); - labelTokenNodeLabelMapping.put(labelToken, List.of(nodeLabel)); - }); - - var propertyBuildersByPropertyKey = nodeSchema.unionProperties().entrySet().stream().collect(toMap( - Map.Entry::getKey, - e -> NodePropertiesFromStoreBuilder.of(e.getValue().defaultValue(), concurrency) - )); - return new NodesBuilder( maxOriginalId, concurrency, - elementIdentifierLabelTokenMapping, - labelTokenNodeLabelMapping, - new ConcurrentHashMap<>(propertyBuildersByPropertyKey), + NodesBuilderContext.fixed(nodeSchema, concurrency), idMapBuilder, hasLabelInformation, nodeSchema.hasProperties(), - deduplicateIds + deduplicateIds, + propertyKey -> nodeSchema.unionProperties().get(propertyKey).state() ); } @@ -213,6 +188,7 @@ public static RelationshipsBuilderBuilder initRelationshipsBuilder() { @Builder.Factory static RelationshipsBuilder relationshipsBuilder( PartialIdMap nodes, + RelationshipType relationshipType, Optional orientation, List propertyConfigs, Optional aggregation, @@ -230,7 +206,6 @@ static RelationshipsBuilder relationshipsBuilder( .map(Aggregation::resolve) .toArray(Aggregation[]::new); - var relationshipType = RelationshipType.ALL_RELATIONSHIPS; var isMultiGraph = Arrays.stream(aggregations).allMatch(Aggregation::equivalentToNone); var actualOrientation = orientation.orElse(Orientation.NATURAL); @@ -285,6 +260,7 @@ static RelationshipsBuilder relationshipsBuilder( .idMap(nodes) .importer(singleTypeRelationshipImporter) .bufferSize(bufferSize) + .relationshipType(relationshipType) .propertyConfigs(propertyConfigs) .isMultiGraph(isMultiGraph) .loadRelationshipProperty(loadRelationshipProperties) @@ -326,17 +302,18 @@ static RelationshipsBuilder relationshipsBuilder( * will be used. */ public static HugeGraph create(IdMap idMap, SingleTypeRelationships relationships) { - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); idMap.availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); relationships.properties().ifPresent(relationshipPropertyStore -> { assert relationshipPropertyStore.values().size() == 1: "Cannot instantiate graph with more than one relationship property."; }); - var relationshipSchema = relationships.relationshipSchema(RelationshipType.of("REL")); + var relationshipSchema = MutableRelationshipSchema.empty(); + relationshipSchema.set(relationships.relationshipSchemaEntry()); return create( - GraphSchema.of(nodeSchema, relationshipSchema, Map.of()), + MutableGraphSchema.of(nodeSchema, relationshipSchema, Map.of()), idMap, Map.of(), relationships diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelToken.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelToken.java index dfc70c6905d..80f2ac61bf7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelToken.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelToken.java @@ -22,6 +22,9 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.NodeLabel; +import java.util.stream.IntStream; +import java.util.stream.Stream; + public interface NodeLabelToken { /** @@ -46,6 +49,13 @@ public interface NodeLabelToken { @NotNull NodeLabel get(int index); String[] getStrings(); + + /** + * @return a stream of {@link org.neo4j.gds.NodeLabel}s represented by this token. + */ + default Stream nodeLabels() { + return IntStream.range(0, this.size()).mapToObj(this::get); + } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java new file mode 100644 index 00000000000..0603ec2308f --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -0,0 +1,213 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.PropertySchema; +import org.neo4j.gds.utils.StringJoining; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +abstract class NodeLabelTokenToPropertyKeys { + + /** + * Creates a thread-safe, mutable mapping. + *

+ * The property schemas are inferred from the input data. + */ + static NodeLabelTokenToPropertyKeys lazy() { + return new Lazy(); + } + + /** + * Creates thread-safe, immutable mapping. + *

+ * The property schemas are inferred from given schema. + */ + static NodeLabelTokenToPropertyKeys fixed(NodeSchema nodeSchema) { + return new Fixed(nodeSchema); + } + + /** + * Assign the given property keys to the given label token. + *

+ * If the token is already present, the property keys are added with set semantics. + */ + abstract void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys); + + /** + * Returns all node labels in this mapping. + */ + abstract Set nodeLabels(); + + /** + * Return the property schemas for the given node label. + */ + abstract Map propertySchemas( + NodeLabel nodeLabelToken, + Map importPropertySchemas + ); + + /** + * Computes the union of the two given mappings without + * changing the contents of the mappings themselves. + */ + static NodeLabelTokenToPropertyKeys union( + NodeLabelTokenToPropertyKeys left, + NodeLabelTokenToPropertyKeys right, + Map importPropertySchemas + ) { + var union = NodeLabelTokenToPropertyKeys.lazy(); + + left.nodeLabels().forEach(nodeLabel -> { + var propertyKeys = left.propertySchemas(nodeLabel, importPropertySchemas).keySet(); + union.add(NodeLabelTokens.ofNodeLabel(nodeLabel), propertyKeys); + }); + right.nodeLabels().forEach(nodeLabel -> { + var propertyKeys = right.propertySchemas(nodeLabel, importPropertySchemas).keySet(); + union.add(NodeLabelTokens.ofNodeLabel(nodeLabel), propertyKeys); + }); + + return union; + } + + private static class Fixed extends NodeLabelTokenToPropertyKeys { + + private final NodeSchema nodeSchema; + + Fixed(NodeSchema nodeSchema) { + this.nodeSchema = nodeSchema; + } + + @Override + void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys) { + // silence is golden + } + + @Override + Set nodeLabels() { + return nodeSchema.availableLabels(); + } + + @Override + Map propertySchemas( + NodeLabel nodeLabel, + Map importPropertySchemas + ) { + var userDefinedPropertySchemas = nodeSchema.get(nodeLabel).properties(); + + // We validate that the property schemas we read during import have + // at least a matching key and a matching type. We cannot do an + // equality check because we cannot infer the default value or the + // property state from just looking at the values. + var overlap = importPropertySchemas + .keySet() + .stream() + .filter(userDefinedPropertySchemas::containsKey) + .collect(Collectors.toSet()); + + if (overlap.size() < userDefinedPropertySchemas.size()) { + var keySet = new HashSet<>(userDefinedPropertySchemas.keySet()); + keySet.removeAll(overlap); + throw new IllegalStateException("Missing node properties during import. " + + "The following keys were part of the schema, " + + "but not contained in the input data: " + + StringJoining.join(keySet) + ); + } + + // We got the same keys and can check the types. + var keysWithIncompatibleTypes = overlap.stream() + .filter(propertyKey -> userDefinedPropertySchemas + .get(propertyKey) + .valueType() != importPropertySchemas + .get(propertyKey) + .valueType()).collect(Collectors.toSet()); + + if (!keysWithIncompatibleTypes.isEmpty()) { + throw new IllegalStateException("Incompatible value types between input schema and input data. " + + "The following keys have incompatible types: " + + StringJoining.join(keysWithIncompatibleTypes) + ); + } + + return userDefinedPropertySchemas; + } + } + + private static class Lazy extends NodeLabelTokenToPropertyKeys { + + private final Map> labelToPropertyKeys; + + Lazy() { + this.labelToPropertyKeys = new HashMap<>(); + } + + @Override + void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys) { + this.labelToPropertyKeys.compute(nodeLabelToken, (token, propertyKeySet) -> { + var keys = (propertyKeySet == null) ? new HashSet() : propertyKeySet; + propertyKeys.forEach(keys::add); + return keys; + }); + } + + @Override + Set nodeLabels() { + return labelToPropertyKeys + .keySet() + .stream() + .map(nodeLabelToken -> nodeLabelToken.isEmpty() ? NodeLabelTokens.ofNodeLabel(NodeLabel.ALL_NODES) : nodeLabelToken) + .flatMap(NodeLabelToken::nodeLabels) + .collect(Collectors.toSet()); + } + + @Override + Map propertySchemas( + NodeLabel nodeLabel, + Map importPropertySchemas + ) { + return labelToPropertyKeys.keySet().stream() + .filter(nodeLabelToken -> { + if (nodeLabelToken.isEmpty() && nodeLabel == NodeLabel.ALL_NODES) { + return true; + } + var nodeLabelTokenCount = nodeLabelToken.size(); + for (int i = 0; i < nodeLabelTokenCount; i++) { + if (nodeLabelToken.get(i).equals(nodeLabel)) { + return true; + } + } + return false; + }) + .flatMap(nodeLabelToken -> this.labelToPropertyKeys.get(nodeLabelToken).stream()) + .collect(Collectors.toMap( + propertyKey -> propertyKey, + importPropertySchemas::get, + (lhs, rhs) -> lhs + )); + } + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 7d156862cc3..9dad02bc635 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -19,25 +19,23 @@ */ package org.neo4j.gds.core.loading.construction; -import com.carrotsearch.hppc.IntObjectHashMap; -import com.carrotsearch.hppc.ObjectIntMap; -import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.compat.LongPropertyReference; +import org.neo4j.gds.compat.PropertyReference; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.LabelInformation; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.NodeImporter; +import org.neo4j.gds.core.loading.NodeImporterBuilder; import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.NodesBatchBuffer; import org.neo4j.gds.core.loading.NodesBatchBufferBuilder; @@ -50,87 +48,66 @@ import org.neo4j.values.virtual.MapValue; import java.util.ArrayList; -import java.util.Collections; +import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.LongAdder; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.function.LongPredicate; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toMap; -import static org.neo4j.gds.core.GraphDimensions.NO_SUCH_LABEL; -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public final class NodesBuilder { - - public static final DefaultValue NO_PROPERTY_VALUE = DefaultValue.DEFAULT; public static final long UNKNOWN_MAX_ID = -1L; private final long maxOriginalId; private final int concurrency; - private int nextLabelId; - private final ObjectIntMap elementIdentifierLabelTokenMapping; private final IdMapBuilder idMapBuilder; + private final Function propertyStates; private final LabelInformation.Builder labelInformationBuilder; - private final IntObjectHashMap> labelTokenNodeLabelMapping; private final LongAdder importedNodes; - private final AutoCloseableThreadLocal threadLocalBuilder; + private final AutoCloseableThreadLocal threadLocalBuilders; private final NodeImporter nodeImporter; - private final Lock lock; - - private final ConcurrentMap propertyBuildersByPropertyKey; - private final boolean hasProperties; + private final NodesBuilderContext nodesBuilderContext; NodesBuilder( long maxOriginalId, int concurrency, - ObjectIntMap elementIdentifierLabelTokenMapping, - IntObjectHashMap> labelTokenNodeLabelMapping, - ConcurrentMap propertyBuildersByPropertyKey, + NodesBuilderContext nodesBuilderContext, IdMapBuilder idMapBuilder, boolean hasLabelInformation, boolean hasProperties, - boolean deduplicateIds + boolean deduplicateIds, + Function propertyStates ) { this.maxOriginalId = maxOriginalId; this.concurrency = concurrency; - this.elementIdentifierLabelTokenMapping = elementIdentifierLabelTokenMapping; + this.nodesBuilderContext = nodesBuilderContext; this.idMapBuilder = idMapBuilder; + this.propertyStates = propertyStates; this.labelInformationBuilder = !hasLabelInformation ? LabelInformationBuilders.allNodes() : LabelInformationBuilders.multiLabelWithCapacity(maxOriginalId + 1); - this.labelTokenNodeLabelMapping = labelTokenNodeLabelMapping; - this.nextLabelId = 0; - this.lock = new ReentrantLock(true); - this.propertyBuildersByPropertyKey = propertyBuildersByPropertyKey; - this.hasProperties = hasProperties; + this.importedNodes = new LongAdder(); - this.nodeImporter = new NodeImporter( - idMapBuilder, - labelInformationBuilder, - labelTokenNodeLabelMapping, - hasProperties - ); + this.nodeImporter = new NodeImporterBuilder() + .idMapBuilder(idMapBuilder) + .labelInformationBuilder(labelInformationBuilder) + .importProperties(hasProperties) + .build(); - Function labelTokenIdFn = elementIdentifierLabelTokenMapping.isEmpty() - ? this::getOrCreateLabelTokenId - : this::getLabelTokenId; - Function propertyBuilderFn = propertyBuildersByPropertyKey.isEmpty() - ? this::getOrCreatePropertyBuilder - : this::getPropertyBuilder; LongPredicate seenNodeIdPredicate = seenNodesPredicate(deduplicateIds, maxOriginalId); long highestPossibleNodeCount = maxOriginalId == UNKNOWN_MAX_ID ? Long.MAX_VALUE : maxOriginalId + 1; - this.threadLocalBuilder = AutoCloseableThreadLocal.withInitial( + + this.threadLocalBuilders = AutoCloseableThreadLocal.withInitial( () -> new NodesBuilder.ThreadLocalBuilder( importedNodes, nodeImporter, @@ -138,8 +115,7 @@ public final class NodesBuilder { seenNodeIdPredicate, hasLabelInformation, hasProperties, - labelTokenIdFn, - propertyBuilderFn + nodesBuilderContext.threadLocalContext() ) ); } @@ -166,7 +142,7 @@ public void addNode(long originalId) { } public void addNode(long originalId, NodeLabelToken nodeLabels) { - this.threadLocalBuilder.get().addNode(originalId, nodeLabels); + this.threadLocalBuilders.get().addNode(originalId, nodeLabels); } public void addNode(long originalId, NodeLabel... nodeLabels) { @@ -198,7 +174,7 @@ public void addNode(long originalId, Map properties, NodeLabel no } public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues properties) { - this.threadLocalBuilder.get().addNode(originalId, nodeLabels, properties); + this.threadLocalBuilders.get().addNode(originalId, nodeLabels, properties); } public long importedNodes() { @@ -210,33 +186,87 @@ public Nodes build() { } public Nodes build(long highestNeoId) { + var localLabelTokenToPropertyKeys = closeThreadLocalBuilders(); + + var idMap = this.idMapBuilder.build(labelInformationBuilder, highestNeoId, concurrency); + var nodeProperties = buildProperties(idMap); + var nodeSchema = buildNodeSchema(idMap, localLabelTokenToPropertyKeys, nodeProperties); + var nodePropertyStore = NodePropertyStore.builder().properties(nodeProperties).build(); + + return ImmutableNodes.builder() + .schema(nodeSchema) + .idMap(idMap) + .properties(nodePropertyStore) + .build(); + } + + private List closeThreadLocalBuilders() { // Flush remaining buffer contents - this.threadLocalBuilder.forEach(ThreadLocalBuilder::flush); + this.threadLocalBuilders.forEach(ThreadLocalBuilder::flush); + // Collect token to property keys for final union + var labelTokenToPropertyKeys = new ArrayList(); + this.threadLocalBuilders.forEach( + threadLocalBuilder -> labelTokenToPropertyKeys.add(threadLocalBuilder.threadLocalContext.nodeLabelTokenToPropertyKeys()) + ); // Clean up resources held by local builders - this.threadLocalBuilder.close(); + this.threadLocalBuilders.close(); - var idMap = this.idMapBuilder.build(labelInformationBuilder, highestNeoId, concurrency); + return labelTokenToPropertyKeys; + } - var nodeImportResultBuilder = ImmutableNodes.builder().idMap(idMap); - if (hasProperties) { - var nodeProperties = buildProperties(idMap); - nodeImportResultBuilder.properties(NodePropertyStore.builder().properties(nodeProperties).build()); - } - return nodeImportResultBuilder.build(); + private MutableNodeSchema buildNodeSchema( + IdMap idMap, + Collection localLabelTokenToPropertyKeys, + Map nodeProperties + ) { + // Collect the property schemas from the imported property values. + var propertyKeysToSchema = nodeProperties + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().propertySchema())); + // Union the label to property key mappings from each import thread. + var globalLabelTokenToPropertyKeys = localLabelTokenToPropertyKeys + .stream() + .reduce( + NodeLabelTokenToPropertyKeys.lazy(), + (left, right) -> NodeLabelTokenToPropertyKeys.union(left, right, propertyKeysToSchema) + ); + // Collect node labels without properties from the id map + // as they are not stored in the above union mapping. + var nodeLabels = new HashSet<>(idMap.availableNodeLabels()); + // Add labels that actually have node properties attached. + localLabelTokenToPropertyKeys.forEach(localMapping -> nodeLabels.addAll(localMapping.nodeLabels())); + + // Use all labels and the global label to property + // key mapping to construct the final node schema. + return nodeLabels.stream() + .reduce( + MutableNodeSchema.empty(), + (unionSchema, nodeLabel) -> unionSchema.addLabel( + nodeLabel, + globalLabelTokenToPropertyKeys.propertySchemas(nodeLabel, propertyKeysToSchema) + ), + (lhs, rhs) -> lhs + ); } private Map buildProperties(IdMap idMap) { - return propertyBuildersByPropertyKey.entrySet().stream().collect(toMap( + return this.nodesBuilderContext.nodePropertyBuilders().entrySet().stream().collect(toMap( Map.Entry::getKey, - entry -> entryToNodeProperty(entry, idMap) + entry -> entryToNodeProperty(entry, propertyStates.apply(entry.getKey()), idMap) )); } - private static NodeProperty entryToNodeProperty(Map.Entry entry, IdMap idMap) { + private static NodeProperty entryToNodeProperty( + Map.Entry entry, + PropertyState propertyState, + IdMap idMap + ) { var nodePropertyValues = entry.getValue().build(idMap); + var valueType = nodePropertyValues.valueType(); return ImmutableNodeProperty.builder() .values(nodePropertyValues) - .propertySchema(PropertySchema.of(entry.getKey(), nodePropertyValues.valueType())) + .propertySchema(PropertySchema.of(entry.getKey(), valueType, valueType.fallbackValue(), propertyState)) .build(); } @@ -244,67 +274,23 @@ private static NodeProperty entryToNodeProperty(Map.Entry * This method must be called in case of an error while using the * NodesBuilder. */ public void close(RuntimeException exception) { - this.threadLocalBuilder.close(); + this.threadLocalBuilders.close(); throw exception; } - private int getOrCreateLabelTokenId(NodeLabel nodeLabel) { - var token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); - if (token == NO_SUCH_LABEL) { - lock.lock(); - token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); - if (token == NO_SUCH_LABEL) { - token = nextLabelId++; - labelTokenNodeLabelMapping.put(token, Collections.singletonList(nodeLabel)); - elementIdentifierLabelTokenMapping.put(nodeLabel, token); - } - lock.unlock(); - } - return token; - } - - private int getLabelTokenId(NodeLabel nodeLabel) { - if (!elementIdentifierLabelTokenMapping.containsKey(nodeLabel)) { - throw new IllegalArgumentException(formatWithLocale("No token was specified for node label %s", nodeLabel)); - } - return elementIdentifierLabelTokenMapping.get(nodeLabel); - } - - private NodePropertiesFromStoreBuilder getOrCreatePropertyBuilder(String propertyKey) { - return propertyBuildersByPropertyKey.computeIfAbsent( - propertyKey, - __ -> NodePropertiesFromStoreBuilder.of(NO_PROPERTY_VALUE, concurrency) - ); - } - - private NodePropertiesFromStoreBuilder getPropertyBuilder(String propertyKey) { - return propertyBuildersByPropertyKey.get(propertyKey); - } - - @ValueClass - public interface IdMapAndProperties { - IdMap idMap(); - - Optional> nodeProperties(); - } - private static class ThreadLocalBuilder implements AutoCloseable { - private static final long NOT_INITIALIZED = -42L; - private final long[] anyLabelArray = {NOT_INITIALIZED}; - private final LongAdder importedNodes; private final LongPredicate seenNodeIdPredicate; private final NodesBatchBuffer buffer; - private final Function labelTokenIdFn; - private final Function propertyBuilderFn; private final NodeImporter nodeImporter; private final List batchNodeProperties; + private final NodesBuilderContext.ThreadLocalContext threadLocalContext; ThreadLocalBuilder( LongAdder importedNodes, @@ -313,13 +299,11 @@ private static class ThreadLocalBuilder implements AutoCloseable { LongPredicate seenNodeIdPredicate, boolean hasLabelInformation, boolean hasProperties, - Function labelTokenIdFn, - Function propertyBuilderFn + NodesBuilderContext.ThreadLocalContext threadLocalContext ) { this.importedNodes = importedNodes; this.seenNodeIdPredicate = seenNodeIdPredicate; - this.labelTokenIdFn = labelTokenIdFn; - this.propertyBuilderFn = propertyBuilderFn; + this.threadLocalContext = threadLocalContext; this.buffer = new NodesBatchBufferBuilder() .capacity(ParallelUtil.DEFAULT_BATCH_SIZE) @@ -327,15 +311,16 @@ private static class ThreadLocalBuilder implements AutoCloseable { .hasLabelInformation(hasLabelInformation) .readProperty(hasProperties) .build(); + this.nodeImporter = nodeImporter; this.batchNodeProperties = new ArrayList<>(buffer.capacity()); } - public void addNode(long originalId, NodeLabelToken nodeLabels) { + public void addNode(long originalId, NodeLabelToken nodeLabelToken) { if (!seenNodeIdPredicate.test(originalId)) { - long[] labels = labelTokens(nodeLabels); + long[] threadLocalTokens = threadLocalContext.addNodeLabelToken(nodeLabelToken); - buffer.add(originalId, LongPropertyReference.empty(), labels); + buffer.add(originalId, LongPropertyReference.empty(), threadLocalTokens); if (buffer.isFull()) { flushBuffer(); reset(); @@ -343,14 +328,16 @@ public void addNode(long originalId, NodeLabelToken nodeLabels) { } } - public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues properties) { + public void addNode(long originalId, NodeLabelToken nodeLabelToken, PropertyValues properties) { if (!seenNodeIdPredicate.test(originalId)) { - long[] labels = labelTokens(nodeLabels); - + long[] threadLocalTokens = threadLocalContext.addNodeLabelTokenAndPropertyKeys( + nodeLabelToken, + properties.propertyKeys() + ); int propertyReference = batchNodeProperties.size(); batchNodeProperties.add(properties); - buffer.add(originalId, LongPropertyReference.of(propertyReference), labels); + buffer.add(originalId, LongPropertyReference.of(propertyReference), threadLocalTokens); if (buffer.isFull()) { flushBuffer(); reset(); @@ -358,72 +345,43 @@ public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues p } } - private long[] labelTokens(NodeLabelToken nodeLabels) { - if (nodeLabels.isEmpty()) { - return anyLabelArray(); - } - - long[] labelIds = new long[nodeLabels.size()]; - for (int i = 0; i < labelIds.length; i++) { - labelIds[i] = labelTokenIdFn.apply(nodeLabels.get(i)); - } - - return labelIds; - } - public void flush() { flushBuffer(); reset(); } + private void reset() { + buffer.reset(); + batchNodeProperties.clear(); + } + private void flushBuffer() { var importedNodesAndProperties = this.nodeImporter.importNodes( - buffer, - (nodeReference, labelIds, propertiesReference) -> { - if (!propertiesReference.isEmpty()) { - var propertyValueIndex = (int) ((LongPropertyReference) propertiesReference).id; - var properties = batchNodeProperties.get(propertyValueIndex); - var importedProperties = new MutableInt(0); - properties.forEach((propertyKey, propertyValue) -> importedProperties.add(importProperty( - nodeReference, - propertyKey, - propertyValue - ))); - return importedProperties.intValue(); - } - return 0; - } + this.buffer, + this.threadLocalContext.threadLocalTokenToNodeLabels(), + this::importProperties ); int importedNodes = RawValues.getHead(importedNodesAndProperties); this.importedNodes.add(importedNodes); } - private void reset() { - buffer.reset(); - batchNodeProperties.clear(); - } + private int importProperties(long nodeReference, long[] labelIds, PropertyReference propertiesReference) { + if (!propertiesReference.isEmpty()) { + var propertyValueIndex = (int) ((LongPropertyReference) propertiesReference).id; + var properties = this.batchNodeProperties.get(propertyValueIndex); - @Override - public void close() {} - - private int importProperty(long neoNodeId, String propertyKey, Value value) { - int propertiesImported = 0; + properties.forEach((propertyKey, propertyValue) -> { + var nodePropertyBuilder = this.threadLocalContext.nodePropertyBuilder(propertyKey); + assert nodePropertyBuilder != null : "observed property key that is not present in schema"; + nodePropertyBuilder.set(nodeReference, propertyValue); - var nodePropertyBuilder = propertyBuilderFn.apply(propertyKey); - if (nodePropertyBuilder != null) { - nodePropertyBuilder.set(neoNodeId, value); - propertiesImported++; + }); + return properties.size(); } - - return propertiesImported; + return 0; } - private long[] anyLabelArray() { - var anyLabelArray = this.anyLabelArray; - if (anyLabelArray[0] == NOT_INITIALIZED) { - anyLabelArray[0] = labelTokenIdFn.apply(NodeLabel.ALL_NODES); - } - return anyLabelArray; - } + @Override + public void close() {} } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java new file mode 100644 index 00000000000..7f05f025038 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import com.carrotsearch.hppc.IntObjectMap; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.core.loading.nodeproperties.NodePropertiesFromStoreBuilder; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; +import java.util.function.Supplier; + +import static java.util.stream.Collectors.toMap; + +final class NodesBuilderContext { + + private static final DefaultValue NO_PROPERTY_VALUE = DefaultValue.DEFAULT; + + // Thread-local mappings that can be computed independently. + private final Supplier tokenToNodeLabelSupplier; + private final Supplier nodeLabelTokenToPropertyKeysSupplier; + // Thread-global mapping as all threads need to write to the same property builders. + private final ConcurrentMap propertyKeyToPropertyBuilder; + + private final int concurrency; + + /** + * Used if no node schema information is available and needs to be inferred from the input data. + */ + static NodesBuilderContext lazy(int concurrency) { + return new NodesBuilderContext( + TokenToNodeLabels::lazy, + NodeLabelTokenToPropertyKeys::lazy, + new ConcurrentHashMap<>(), + concurrency + ); + } + + /** + * Used if a node schema is available upfront. + */ + static NodesBuilderContext fixed(NodeSchema nodeSchema, int concurrency) { + var propertyBuildersByPropertyKey = nodeSchema.unionProperties().entrySet().stream().collect(toMap( + Map.Entry::getKey, + e -> NodePropertiesFromStoreBuilder.of(e.getValue().defaultValue(), concurrency) + )); + + return new NodesBuilderContext( + () -> TokenToNodeLabels.fixed(nodeSchema.availableLabels()), + () -> NodeLabelTokenToPropertyKeys.fixed(nodeSchema), + new ConcurrentHashMap<>(propertyBuildersByPropertyKey), + concurrency + ); + } + + Map nodePropertyBuilders() { + return this.propertyKeyToPropertyBuilder; + } + + private NodesBuilderContext( + Supplier tokenToNodeLabelSupplier, + Supplier nodeLabelTokenToPropertyKeysSupplier, + ConcurrentMap propertyKeyToPropertyBuilder, + int concurrency + ) { + this.tokenToNodeLabelSupplier = tokenToNodeLabelSupplier; + this.nodeLabelTokenToPropertyKeysSupplier = nodeLabelTokenToPropertyKeysSupplier; + this.propertyKeyToPropertyBuilder = propertyKeyToPropertyBuilder; + this.concurrency = concurrency; + } + + ThreadLocalContext threadLocalContext() { + Function propertyBuilderFn = this.propertyKeyToPropertyBuilder.isEmpty() + ? this::getOrCreatePropertyBuilder + : this::getPropertyBuilder; + + return new ThreadLocalContext( + tokenToNodeLabelSupplier.get(), + nodeLabelTokenToPropertyKeysSupplier.get(), + propertyBuilderFn + ); + } + + private NodePropertiesFromStoreBuilder getOrCreatePropertyBuilder(String propertyKey) { + return this.propertyKeyToPropertyBuilder.computeIfAbsent( + propertyKey, + __ -> NodePropertiesFromStoreBuilder.of(NO_PROPERTY_VALUE, concurrency) + ); + } + + private NodePropertiesFromStoreBuilder getPropertyBuilder(String propertyKey) { + return this.propertyKeyToPropertyBuilder.get(propertyKey); + } + + static class ThreadLocalContext { + + private static final long NOT_INITIALIZED = -42L; + + private final long[] anyLabelArray = {NOT_INITIALIZED}; + private final TokenToNodeLabels tokenToNodeLabels; + private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; + private final Function propertyBuilderFn; + + ThreadLocalContext( + TokenToNodeLabels tokenToNodeLabels, + NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, + Function propertyBuilderFn + ) { + this.tokenToNodeLabels = tokenToNodeLabels; + this.nodeLabelTokenToPropertyKeys = nodeLabelTokenToPropertyKeys; + this.propertyBuilderFn = propertyBuilderFn; + } + + NodePropertiesFromStoreBuilder nodePropertyBuilder(String propertyKey) { + return this.propertyBuilderFn.apply(propertyKey); + } + + NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys() { + return this.nodeLabelTokenToPropertyKeys; + } + + IntObjectMap> threadLocalTokenToNodeLabels() { + return this.tokenToNodeLabels.labelTokenNodeLabelMapping(); + } + + long[] addNodeLabelToken(NodeLabelToken nodeLabelToken) { + return getOrCreateLabelTokens(nodeLabelToken); + } + + long[] addNodeLabelTokenAndPropertyKeys(NodeLabelToken nodeLabelToken, Iterable propertyKeys) { + long[] tokens = getOrCreateLabelTokens(nodeLabelToken); + this.nodeLabelTokenToPropertyKeys.add(nodeLabelToken, propertyKeys); + return tokens; + } + + private long[] getOrCreateLabelTokens(NodeLabelToken nodeLabelToken) { + if (nodeLabelToken.isEmpty()) { + return anyLabelArray(); + } + + long[] labelIds = new long[nodeLabelToken.size()]; + for (int i = 0; i < labelIds.length; i++) { + labelIds[i] = this.tokenToNodeLabels.getOrCreateToken(nodeLabelToken.get(i)); + } + + return labelIds; + } + + private long[] anyLabelArray() { + var anyLabelArray = this.anyLabelArray; + if (anyLabelArray[0] == NOT_INITIALIZED) { + anyLabelArray[0] = tokenToNodeLabels.getOrCreateToken(NodeLabel.ALL_NODES); + } + return anyLabelArray; + } + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java index d018bf817ce..8dda9c87ed5 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java @@ -25,6 +25,7 @@ import org.neo4j.values.virtual.MapValue; import java.util.Map; +import java.util.Set; import java.util.function.BiConsumer; public abstract class PropertyValues { @@ -33,6 +34,10 @@ public abstract class PropertyValues { public abstract boolean isEmpty(); + public abstract int size(); + + public abstract Iterable propertyKeys(); + public static PropertyValues of(MapValue mapValue) { return new CypherPropertyValues(mapValue); } @@ -57,6 +62,16 @@ public void forEach(BiConsumer consumer) { public boolean isEmpty() { return this.properties.isEmpty(); } + + @Override + public int size() { + return this.properties.size(); + } + + @Override + public Set propertyKeys() { + return this.properties.keySet(); + } } private static final class CypherPropertyValues extends PropertyValues { @@ -79,5 +94,15 @@ public void forEach(BiConsumer consumer) { public boolean isEmpty() { return this.properties.isEmpty(); } + + @Override + public int size() { + return this.properties.size(); + } + + @Override + public Iterable propertyKeys() { + return this.properties.keySet(); + } } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java index 037203c9089..f6d6119b9fc 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java @@ -20,6 +20,7 @@ package org.neo4j.gds.core.loading.construction; import org.immutables.builder.Builder; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.ImmutableProperties; import org.neo4j.gds.api.ImmutableRelationshipProperty; @@ -29,6 +30,8 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.ImmutableRelationshipPropertySchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; +import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.core.compress.AdjacencyCompressor; import org.neo4j.gds.core.compress.AdjacencyListsWithProperties; import org.neo4j.gds.core.concurrency.RunWithConcurrency; @@ -47,6 +50,7 @@ abstract class SingleTypeRelationshipsBuilder { final PartialIdMap idMap; final int bufferSize; + final RelationshipType relationshipType; final List propertyConfigs; final boolean isMultiGraph; @@ -62,6 +66,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( SingleTypeRelationshipImporter importer, Optional inverseImporter, int bufferSize, + RelationshipType relationshipType, List propertyConfigs, boolean isMultiGraph, boolean loadRelationshipProperty, @@ -75,6 +80,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( importer, inverseImporter.get(), bufferSize, + relationshipType, propertyConfigs, isMultiGraph, loadRelationshipProperty, @@ -86,6 +92,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( idMap, importer, bufferSize, + relationshipType, propertyConfigs, isMultiGraph, loadRelationshipProperty, @@ -98,6 +105,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( SingleTypeRelationshipsBuilder( PartialIdMap idMap, int bufferSize, + RelationshipType relationshipType, List propertyConfigs, boolean isMultiGraph, boolean loadRelationshipProperty, @@ -107,6 +115,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( ) { this.idMap = idMap; this.bufferSize = bufferSize; + this.relationshipType = relationshipType; this.propertyConfigs = propertyConfigs; this.isMultiGraph = isMultiGraph; this.loadRelationshipProperty = loadRelationshipProperty; @@ -143,6 +152,28 @@ SingleTypeRelationships build( return singleTypeRelationshipImportResult(); } + MutableRelationshipSchemaEntry relationshipSchemaEntry(Optional properties) { + var entry = new MutableRelationshipSchemaEntry( + relationshipType, + direction + ); + + properties.ifPresent(relationshipPropertyStore -> relationshipPropertyStore + .relationshipProperties() + .forEach((propertyKey, relationshipProperty) -> entry.addProperty( + propertyKey, + RelationshipPropertySchema.of(propertyKey, + relationshipProperty.valueType(), + relationshipProperty.defaultValue(), + relationshipProperty.propertyState(), + relationshipProperty.aggregation() + ) + )) + ); + + return entry; + } + RelationshipPropertyStore relationshipPropertyStore(AdjacencyListsWithProperties adjacencyListsWithProperties) { var propertyStoreBuilder = RelationshipPropertyStore.builder(); @@ -186,6 +217,7 @@ static class NonIndexed extends SingleTypeRelationshipsBuilder { PartialIdMap idMap, SingleTypeRelationshipImporter importer, int bufferSize, + RelationshipType relationshipType, List propertyConfigs, boolean isMultiGraph, boolean loadRelationshipProperty, @@ -196,6 +228,7 @@ static class NonIndexed extends SingleTypeRelationshipsBuilder { super( idMap, bufferSize, + relationshipType, propertyConfigs, isMultiGraph, loadRelationshipProperty, @@ -231,15 +264,17 @@ SingleTypeRelationships singleTypeRelationshipImportResult() { .elementCount(relationshipCount) .build(); - var singleRelationshipTypeImportResultBuilder = SingleTypeRelationships.builder() - .topology(topology) - .direction(this.direction); + var singleRelationshipTypeImportResultBuilder = SingleTypeRelationships.builder().topology(topology); + RelationshipPropertyStore properties = null; if (loadRelationshipProperty) { - var properties = relationshipPropertyStore(adjacencyListsWithProperties); + properties = relationshipPropertyStore(adjacencyListsWithProperties); singleRelationshipTypeImportResultBuilder.properties(properties); } + singleRelationshipTypeImportResultBuilder + .relationshipSchemaEntry(relationshipSchemaEntry(Optional.ofNullable(properties))); + return singleRelationshipTypeImportResultBuilder.build(); } } @@ -254,6 +289,7 @@ static class Indexed extends SingleTypeRelationshipsBuilder { SingleTypeRelationshipImporter forwardImporter, SingleTypeRelationshipImporter inverseImporter, int bufferSize, + RelationshipType relationshipType, List propertyConfigs, boolean isMultiGraph, boolean loadRelationshipProperty, @@ -264,6 +300,7 @@ static class Indexed extends SingleTypeRelationshipsBuilder { super( idMap, bufferSize, + relationshipType, propertyConfigs, isMultiGraph, loadRelationshipProperty, @@ -316,15 +353,18 @@ SingleTypeRelationships singleTypeRelationshipImportResult() { var singleRelationshipTypeImportResultBuilder = SingleTypeRelationships.builder() .topology(forwardTopology) - .inverseTopology(inverseTopology) - .direction(this.direction); + .inverseTopology(inverseTopology); + RelationshipPropertyStore forwardProperties = null; if (loadRelationshipProperty) { - var forwardProperties = relationshipPropertyStore(forwardListWithProperties); + forwardProperties = relationshipPropertyStore(forwardListWithProperties); var inverseProperties = relationshipPropertyStore(inverseListWithProperties); singleRelationshipTypeImportResultBuilder.properties(forwardProperties).inverseProperties(inverseProperties); } + singleRelationshipTypeImportResultBuilder + .relationshipSchemaEntry(relationshipSchemaEntry(Optional.ofNullable(forwardProperties))); + return singleRelationshipTypeImportResultBuilder.build(); } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabels.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabels.java new file mode 100644 index 00000000000..376bd96c379 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabels.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import com.carrotsearch.hppc.IntObjectHashMap; +import com.carrotsearch.hppc.ObjectIntMap; +import com.carrotsearch.hppc.ObjectIntScatterMap; +import org.apache.commons.lang3.mutable.MutableInt; +import org.neo4j.gds.NodeLabel; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.neo4j.gds.core.GraphDimensions.ANY_LABEL; +import static org.neo4j.gds.core.GraphDimensions.NO_SUCH_LABEL; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +abstract class TokenToNodeLabels { + + final ObjectIntMap nodeLabelToLabelTokenMap; + final IntObjectHashMap> labelTokenToNodeLabelMap; + + static TokenToNodeLabels fixed(Collection nodeLabels) { + var elementIdentifierLabelTokenMapping = new ObjectIntScatterMap(); + var labelTokenNodeLabelMapping = new IntObjectHashMap>(); + var labelTokenCounter = new MutableInt(0); + nodeLabels.forEach(nodeLabel -> { + int labelToken = nodeLabel == NodeLabel.ALL_NODES + ? ANY_LABEL + : labelTokenCounter.getAndIncrement(); + + elementIdentifierLabelTokenMapping.put(nodeLabel, labelToken); + labelTokenNodeLabelMapping.put(labelToken, List.of(nodeLabel)); + }); + + return new Fixed(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); + } + + static TokenToNodeLabels lazy() { + return new Lazy(); + } + + private TokenToNodeLabels() { + this.nodeLabelToLabelTokenMap = new ObjectIntScatterMap<>(); + this.labelTokenToNodeLabelMap = new IntObjectHashMap<>(); + } + + private TokenToNodeLabels( + ObjectIntMap nodeLabelToLabelTokenMap, + IntObjectHashMap> labelTokenToNodeLabelMap + ) { + this.nodeLabelToLabelTokenMap = nodeLabelToLabelTokenMap; + this.labelTokenToNodeLabelMap = labelTokenToNodeLabelMap; + } + + IntObjectHashMap> labelTokenNodeLabelMapping() { + return this.labelTokenToNodeLabelMap; + } + + abstract int getOrCreateToken(NodeLabel nodeLabel); + + private static final class Fixed extends TokenToNodeLabels { + + private Fixed( + ObjectIntMap elementIdentifierLabelTokenMapping, + IntObjectHashMap> labelTokenNodeLabelMapping + ) { + super(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); + } + + @Override + public int getOrCreateToken(NodeLabel nodeLabel) { + if (!nodeLabelToLabelTokenMap.containsKey(nodeLabel)) { + throw new IllegalArgumentException(formatWithLocale("No token was specified for node label %s", nodeLabel)); + } + return nodeLabelToLabelTokenMap.get(nodeLabel); + } + } + + private static final class Lazy extends TokenToNodeLabels { + + private int nextLabelId; + + private Lazy() { + this.nextLabelId = 0; + } + + @Override + public int getOrCreateToken(NodeLabel nodeLabel) { + var token = nodeLabelToLabelTokenMap.getOrDefault(nodeLabel, NO_SUCH_LABEL); + if (token == NO_SUCH_LABEL) { + token = nextLabelId++; + labelTokenToNodeLabelMap.put(token, Collections.singletonList(nodeLabel)); + nodeLabelToLabelTokenMap.put(nodeLabel, token); + } + return token; + } + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java index 1ffc191853e..1bb7ac3fbbb 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java @@ -116,7 +116,7 @@ public double[] doubleArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java index 709830e4eef..e5b45a1e484 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java @@ -120,7 +120,7 @@ public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highes var propertyValues = propertiesByMappedIdsBuilder.build(); - var maybeMaxValue = size > 0 + var maybeMaxValue = propertyValues.capacity() > 0 ? OptionalDouble.of((double) MAX_VALUE.getVolatile(DoubleNodePropertiesBuilder.this)) : OptionalDouble.empty(); @@ -183,7 +183,7 @@ public OptionalDouble getMaxDoublePropertyValue() { } @Override - public long size() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java index 8c5af28a964..86633552e96 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java @@ -114,7 +114,7 @@ public float[] floatArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java index 86fd085ed7f..6c3616b641a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java @@ -119,7 +119,7 @@ public long[] longArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java index 25b0d77b20b..e743d718d13 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java @@ -128,7 +128,7 @@ public NodePropertyValues build(long size, PartialIdMap idMap, long highestOrigi var propertyValues = propertiesByMappedIdsBuilder.build(); - var maybeMaxValue = size > 0 + var maybeMaxValue = propertyValues.capacity() > 0 ? OptionalLong.of((long) MAX_VALUE.getVolatile(LongNodePropertiesBuilder.this)) : OptionalLong.empty(); @@ -191,7 +191,7 @@ public OptionalLong getMaxLongPropertyValue() { } @Override - public long size() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java index e9cf0ce61de..df891258ee3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java @@ -31,7 +31,6 @@ import org.neo4j.values.storable.Values; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.atomic.LongAdder; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; import static org.neo4j.values.storable.Values.NO_VALUE; @@ -63,7 +62,6 @@ public static NodePropertiesFromStoreBuilder of( private final DefaultValue defaultValue; private final int concurrency; private final AtomicReference innerBuilder; - private final LongAdder size; private NodePropertiesFromStoreBuilder( DefaultValue defaultValue, @@ -72,7 +70,6 @@ private NodePropertiesFromStoreBuilder( this.defaultValue = defaultValue; this.concurrency = concurrency; this.innerBuilder = new AtomicReference<>(); - this.size = new LongAdder(); } public void set(long neoNodeId, Value value) { @@ -81,7 +78,6 @@ public void set(long neoNodeId, Value value) { initializeWithType(value); } innerBuilder.get().setValue(neoNodeId, value); - size.increment(); } } @@ -94,7 +90,7 @@ public NodePropertyValues build(IdMap idMap) { } } - return innerBuilder.get().build(this.size.sum(), idMap, idMap.highestOriginalId()); + return innerBuilder.get().build(idMap.nodeCount(), idMap, idMap.highestOriginalId()); } // This is synchronized as we want to prevent the creation of multiple InnerNodePropertiesBuilders of which only once survives. diff --git a/core/src/main/java/org/neo4j/gds/core/utils/IdentityPropertyValues.java b/core/src/main/java/org/neo4j/gds/core/utils/IdentityPropertyValues.java index 0d52db2e44d..1f0d0af645a 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/IdentityPropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/IdentityPropertyValues.java @@ -34,7 +34,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return expectedPropertyCount; } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/OriginalIdNodePropertyValues.java b/core/src/main/java/org/neo4j/gds/core/utils/OriginalIdNodePropertyValues.java index 38666202346..afcd5d5806a 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/OriginalIdNodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/OriginalIdNodePropertyValues.java @@ -42,7 +42,7 @@ public OptionalLong getMaxLongPropertyValue() { } @Override - public long size() { + public long nodeCount() { return idMap.nodeCount(); } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java b/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java new file mode 100644 index 00000000000..94b91772f8f --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.utils; + +import com.carrotsearch.hppc.sorting.IndirectComparator; +import com.carrotsearch.hppc.sorting.IndirectSort; + +public final class TwoArraysSort { + private TwoArraysSort() {} + /** + * Sort two arrays simultaneously based on values of the first (long) array. + * E.g. {[4, 1, 8], [0.5, 1.9, 0.9]} yields {[1, 4, 8], [1,9, 0.5, 0.9]} + * + * @param longArray Array of long values (e.g. neighbours ids) + * @param doubleArray Array of double values (e.g. neighbours weighs) + * @param length Number of values to sort + */ + public static void sortDoubleArrayByLongValues(long[] longArray, double[] doubleArray, int length) { + assert longArray.length >= length; + assert doubleArray.length >= length; + var order = IndirectSort.mergesort(0, length, new AscendingLongComparator(longArray)); + reorder(order, longArray, doubleArray, length); + } + + private static void reorder(int[] order, long[] longArray, double[] doubleArray, int length) { + for (int i = 0; i < length; i++) { + while (order[i] != i) { + int idx = order[order[i]]; + var longVal = longArray[order[i]]; + var doubleVal = doubleArray[order[i]]; + + longArray[order[i]] = longArray[i]; + doubleArray[order[i]] = doubleArray[i]; + order[order[i]] = order[i]; + + order[i] = idx; + longArray[i] = longVal; + doubleArray[i] = doubleVal; + } + } + } + + public static class AscendingLongComparator implements IndirectComparator { + private final long[] array; + + AscendingLongComparator(long[] array) { + this.array = array; + } + + public int compare(int indexA, int indexB) { + final long a = array[indexA]; + final long b = array[indexB]; + + if (a < b) + return -1; + if (a > b) + return 1; + return 0; + } + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicDoubleArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicDoubleArray.java index 320e2f07c89..922165f2993 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicDoubleArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicDoubleArray.java @@ -175,7 +175,7 @@ public double doubleValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeAtomicDoubleArray.this.size(); } }; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicLongArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicLongArray.java index 593e0f5a1b4..e79b4daae0a 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicLongArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicLongArray.java @@ -204,7 +204,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeAtomicLongArray.this.size(); } }; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeByteArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeByteArray.java index d2cf4588179..4faefaba1b0 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeByteArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeByteArray.java @@ -205,7 +205,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeByteArray.this.size(); } }; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeDoubleArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeDoubleArray.java index 4d53c6d5a0c..b0c228fd89e 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeDoubleArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeDoubleArray.java @@ -189,7 +189,7 @@ public double doubleValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeDoubleArray.this.size(); } }; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeIntArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeIntArray.java index 05752c5e733..b029ae82c22 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeIntArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeIntArray.java @@ -205,7 +205,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeIntArray.this.size(); } }; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeLongArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeLongArray.java index d84ef5cf64d..6806830798c 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeLongArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeLongArray.java @@ -207,7 +207,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeLongArray.this.size(); } }; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeObjectArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeObjectArray.java index 274e8d723a7..b4b1fb3fc41 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeObjectArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeObjectArray.java @@ -190,7 +190,7 @@ public float[] floatArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeObjectArray.this.size(); } }; @@ -203,7 +203,7 @@ public double[] doubleArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeObjectArray.this.size(); } }; @@ -216,7 +216,7 @@ public long[] longArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return HugeObjectArray.this.size(); } }; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java index 1a2d88609f5..62c0f10ec22 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java @@ -213,6 +213,11 @@ public static final class Builder { .toArray(Shard[]::new); } + /** + * Add a node to the mapping. + * @return {@code mappedId >= 0} if the node was added, + * or {@code -(mappedId) - 1} if the node was already mapped. + */ public long addNode(long nodeId) { var shard = findShard(nodeId, this.shards, this.shardShift, this.shardMask); try (var ignoredLock = shard.acquireLock()) { @@ -220,11 +225,6 @@ public long addNode(long nodeId) { } } - public long toMappedNodeId(long nodeId) { - var shard = findShard(nodeId, this.shards, this.shardShift, this.shardMask); - return shard.toMappedNodeId(nodeId); - } - public ShardedLongLongMap build() { return ShardedLongLongMap.build( this.nodeCount.get(), @@ -252,15 +252,15 @@ private Shard(AtomicLong nextId) { this.nextId = nextId; } - long toMappedNodeId(long nodeId) { - return mapping.getIfAbsent(nodeId, IdMap.NOT_FOUND); - } - long addNode(long nodeId) { this.assertIsUnderLock(); - long internalId = this.nextId.getAndIncrement(); - mapping.put(nodeId, internalId); - return internalId; + long mappedId = mapping.getIfAbsent(nodeId, IdMap.NOT_FOUND); + if (mappedId != IdMap.NOT_FOUND) { + return -mappedId - 1; + } + mappedId = nextId.getAndIncrement(); + mapping.put(nodeId, mappedId); + return mappedId; } } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/dss/DisjointSetStruct.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/dss/DisjointSetStruct.java index 392afb21fd7..11ae9672912 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/dss/DisjointSetStruct.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/dss/DisjointSetStruct.java @@ -77,7 +77,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return DisjointSetStruct.this.size(); } }; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressLogger.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressLogger.java index 6822ce40229..b08a92330a0 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressLogger.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressLogger.java @@ -29,12 +29,18 @@ class TaskProgressLogger extends BatchingProgressLogger { private final Task baseTask; - private final LoggingLeafTaskVisitor loggingLeafTaskVisitor; + private final TaskVisitor loggingLeafTaskVisitor; TaskProgressLogger(Log log, Task baseTask, int concurrency) { super(log, baseTask, concurrency); this.baseTask = baseTask; this.loggingLeafTaskVisitor = new LoggingLeafTaskVisitor(this); + + } + TaskProgressLogger(Log log, Task baseTask, int concurrency, TaskVisitor leafTaskVisitor) { + super(log, baseTask, concurrency); + this.baseTask = baseTask; + this.loggingLeafTaskVisitor = leafTaskVisitor; } void logBeginSubTask(Task task, Task parentTask) { diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java index 7668569cb45..cb2a7c43bac 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java @@ -51,19 +51,33 @@ public class TaskProgressTracker implements ProgressTracker { private long currentTotalSteps; private double progressLeftOvers; - private Runnable onError; + private final Runnable onError; public TaskProgressTracker(Task baseTask, Log log, int concurrency, TaskRegistryFactory taskRegistryFactory) { this(baseTask, log, concurrency, new JobId(), taskRegistryFactory, EmptyUserLogRegistryFactory.INSTANCE); } public TaskProgressTracker( - Task baseTask, Log log, int concurrency, JobId jobId, TaskRegistryFactory taskRegistryFactory, + Task baseTask, + Log log, + int concurrency, + JobId jobId, + TaskRegistryFactory taskRegistryFactory, + UserLogRegistryFactory userLogRegistryFactory + ) { + this(baseTask, jobId, taskRegistryFactory, new TaskProgressLogger(log, baseTask, concurrency), userLogRegistryFactory); + } + + protected TaskProgressTracker( + Task baseTask, + JobId jobId, + TaskRegistryFactory taskRegistryFactory, + TaskProgressLogger taskProgressLogger, UserLogRegistryFactory userLogRegistryFactory ) { this.baseTask = baseTask; this.taskRegistry = taskRegistryFactory.newInstance(jobId); - this.taskProgressLogger = new TaskProgressLogger(log, baseTask, concurrency); + this.taskProgressLogger = taskProgressLogger; this.currentTask = Optional.empty(); this.currentTotalSteps = UNKNOWN_STEPS; this.progressLeftOvers = 0; @@ -77,7 +91,7 @@ public TaskProgressTracker( AtomicBoolean didLog = new AtomicBoolean(false); this.onError = () -> { if (!didLog.get()) { - taskProgressLogger.logError("Tried to log progress, but there are no running tasks being tracked"); + taskProgressLogger.logWarning(":: Tried to log progress, but there are no running tasks being tracked"); didLog.set(true); } }; @@ -253,9 +267,9 @@ public void endSubTaskWithFailure(String expectedTaskDescription) { } @TestOnly - public Task currentSubTask() { + Task currentSubTask() { requireCurrentTask(); - return currentTask.get(); + return currentTask.orElseThrow(); } @Nullable diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java new file mode 100644 index 00000000000..38c7d57f442 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.utils.progress.tasks; + +import org.neo4j.gds.core.utils.progress.JobId; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.core.utils.warnings.UserLogRegistryFactory; +import org.neo4j.logging.Log; + +public final class TaskTreeProgressTracker extends TaskProgressTracker { + + public TaskTreeProgressTracker( + Task baseTask, + Log log, + int concurrency, + JobId jobId, + TaskRegistryFactory taskRegistryFactory, + UserLogRegistryFactory userLogRegistryFactory + ) { + super( + baseTask, + jobId, + taskRegistryFactory, + new TaskProgressLogger( + log, + baseTask, + concurrency, + new PassThroughTaskVisitor() + ), + userLogRegistryFactory + ); + } + + @Override + public void logSteps(long steps) { + // NOOP + } + + @Override + public void logProgress(long value) { + // NOOP + } + + @Override + public void logProgress(long value, String messageTemplate) { + // NOOP + } + + private static class PassThroughTaskVisitor implements TaskVisitor { + @Override + public void visitLeafTask(LeafTask leafTask) { + // NOOP --> just pass through + } + } +} diff --git a/core/src/main/java/org/neo4j/gds/utils/ExceptionUtil.java b/core/src/main/java/org/neo4j/gds/utils/ExceptionUtil.java index aa20a459858..151c7414236 100644 --- a/core/src/main/java/org/neo4j/gds/utils/ExceptionUtil.java +++ b/core/src/main/java/org/neo4j/gds/utils/ExceptionUtil.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.Nullable; +import org.neo4j.logging.Log; import java.io.IOException; import java.io.UncheckedIOException; @@ -185,6 +186,14 @@ public static void run(CheckedRunnable runnable) { runnable.run(); } + public static void safeRunWithLogException(Log log, Supplier message, Runnable runnable) { + try { + runnable.run(); + } catch (Exception e) { + log.warn(message.get(), e); + } + } + public static Consumer consumer(CheckedConsumer consumer) { return consumer; } diff --git a/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java b/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java index 769fba4c0a0..f2ec1e3bdf3 100644 --- a/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java +++ b/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java @@ -185,9 +185,21 @@ private static Stream values() { Arguments.of(42, (Function) DefaultValue::longValue, 42L), Arguments.of(42, (Function) DefaultValue::doubleValue, 42D), Arguments.of(13.37, (Function) DefaultValue::doubleValue, 13.37D), - Arguments.of(List.of(13.37, 42), (Function) DefaultValue::doubleArrayValue, new double[] {13.37D, 42D}), - Arguments.of(List.of(1337L, 42L), (Function) DefaultValue::longArrayValue, new long[] {1337L, 42L}), - Arguments.of(List.of(1337, 42), (Function) DefaultValue::longArrayValue, new long[] {1337L, 42L}) + Arguments.of( + List.of(13.37, 42), + (Function) DefaultValue::doubleArrayValue, + new double[]{13.37D, 42D} + ), + Arguments.of( + List.of(1337L, 42L), + (Function) DefaultValue::longArrayValue, + new long[]{1337L, 42L} + ), + Arguments.of( + List.of(1337, 42), + (Function) DefaultValue::longArrayValue, + new long[]{1337L, 42L} + ) ); } diff --git a/core/src/test/java/org/neo4j/gds/api/schema/GraphSchemaIntegrationTest.java b/core/src/test/java/org/neo4j/gds/api/schema/GraphSchemaIntegrationTest.java index aac7189122b..e5e25f6ef32 100644 --- a/core/src/test/java/org/neo4j/gds/api/schema/GraphSchemaIntegrationTest.java +++ b/core/src/test/java/org/neo4j/gds/api/schema/GraphSchemaIntegrationTest.java @@ -27,7 +27,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.StoreLoaderBuilder; @@ -37,7 +36,6 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.core.Aggregation; -import java.util.List; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -62,10 +60,10 @@ void computesCorrectNodeSchema(PropertySchema expectedSchema, PropertyMapping pr Graph graph = new StoreLoaderBuilder() .databaseService(db) .addNodeProjection( - NodeProjection.of( - "Node", - PropertyMappings.of(List.of(propertyMapping)) - ) + NodeProjection.builder() + .label("Node") + .addProperty(propertyMapping) + .build() ) .build() .graph(); @@ -81,7 +79,7 @@ void computesCorrectRelationshipSchema(RelationshipPropertySchema expectedSchema .addRelationshipProjection( RelationshipProjection.builder() .type("REL") - .properties(PropertyMappings.of(List.of(propertyMapping))) + .addProperties(propertyMapping) .build() ) .build() diff --git a/core/src/test/java/org/neo4j/gds/beta/generator/RandomGraphGeneratorTest.java b/core/src/test/java/org/neo4j/gds/beta/generator/RandomGraphGeneratorTest.java index aedf8207eb3..5914909d566 100644 --- a/core/src/test/java/org/neo4j/gds/beta/generator/RandomGraphGeneratorTest.java +++ b/core/src/test/java/org/neo4j/gds/beta/generator/RandomGraphGeneratorTest.java @@ -31,8 +31,8 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.config.RandomGraphGeneratorConfig; import org.neo4j.gds.config.RandomGraphGeneratorConfig.AllowSelfLoops; import org.neo4j.gds.core.Aggregation; @@ -465,8 +465,8 @@ void shouldProduceCorrectRelationshipCountWithAggregation() { void shouldProduceCorrectSchema( String name, RandomGraphGenerator generator, - NodeSchema expectedNodeSchema, - RelationshipSchema expectedRelationshipSchema + MutableNodeSchema expectedNodeSchema, + MutableRelationshipSchema expectedRelationshipSchema ) { var graph = generator.generate(); @@ -518,8 +518,8 @@ static Stream expectedSchemas() { .averageDegree(1) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.ALL_NODES), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.ALL_NODES), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) ), Arguments.of("node label", RandomGraphGenerator .builder() @@ -528,8 +528,8 @@ static Stream expectedSchemas() { .nodeLabelProducer(nodeId -> NodeLabelTokens.ofNodeLabels(NodeLabel.of("A"))) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) ), Arguments.of("relationship type", RandomGraphGenerator .builder() @@ -538,8 +538,8 @@ static Stream expectedSchemas() { .relationshipType(RelationshipType.of("FOOBAR")) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.ALL_NODES), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("FOOBAR"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.ALL_NODES), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("FOOBAR"), Direction.DIRECTED) ), Arguments.of("node label and relationship type", RandomGraphGenerator .builder() @@ -549,8 +549,8 @@ static Stream expectedSchemas() { .relationshipType(RelationshipType.of("FOOBAR")) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("FOOBAR"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("FOOBAR"), Direction.DIRECTED) ), Arguments.of("node label and node property", RandomGraphGenerator .builder() @@ -560,8 +560,8 @@ static Stream expectedSchemas() { .nodePropertyProducer(PropertyProducer.randomLong("nodeProp", 0, 42)) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")).addProperty(NodeLabel.of("A"), "nodeProp", ValueType.LONG), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")).addProperty(NodeLabel.of("A"), "nodeProp", ValueType.LONG), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) ), Arguments.of("relationship type and node property", RandomGraphGenerator .builder() @@ -571,8 +571,8 @@ static Stream expectedSchemas() { .relationshipPropertyProducer(PropertyProducer.randomDouble("relProperty", 0, 42)) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.ALL_NODES), - RelationshipSchema.empty().addProperty(RelationshipType.of("FOOBAR"), + MutableNodeSchema.empty().addLabel(NodeLabel.ALL_NODES), + MutableRelationshipSchema.empty().addProperty(RelationshipType.of("FOOBAR"), Direction.DIRECTED, "relProperty", ValueType.DOUBLE, @@ -589,8 +589,8 @@ static Stream expectedSchemas() { .relationshipPropertyProducer(PropertyProducer.randomDouble("relProp", 0, 42)) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")).addProperty(NodeLabel.of("A"), "nodeProp", ValueType.LONG), - RelationshipSchema.empty().addProperty(RelationshipType.of("FOOBAR"), + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")).addProperty(NodeLabel.of("A"), "nodeProp", ValueType.LONG), + MutableRelationshipSchema.empty().addProperty(RelationshipType.of("FOOBAR"), Direction.DIRECTED, "relProp", ValueType.DOUBLE, @@ -607,8 +607,8 @@ static Stream expectedSchemas() { .relationshipDistribution(RelationshipDistribution.RANDOM) .direction(UNDIRECTED) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")), - RelationshipSchema + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")), + MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("FOOBAR"), UNDIRECTED, diff --git a/core/src/test/java/org/neo4j/gds/config/GraphProjectFromCypherConfigTest.java b/core/src/test/java/org/neo4j/gds/config/GraphProjectFromCypherConfigTest.java index 6a6bdb062a4..bc013a0044f 100644 --- a/core/src/test/java/org/neo4j/gds/config/GraphProjectFromCypherConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/GraphProjectFromCypherConfigTest.java @@ -24,8 +24,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.AbstractProjections; +import org.neo4j.gds.ImmutableNodeProjections; import org.neo4j.gds.ImmutableRelationshipProjections; -import org.neo4j.gds.NodeProjections; import org.neo4j.gds.core.CypherMapWrapper; import java.util.Map; @@ -71,9 +71,9 @@ void omitCypherParametersFromToMap() { static Stream invalidKeys() { return Stream.of( - Arguments.of(GraphProjectFromStoreConfig.NODE_PROJECTION_KEY, NodeProjections.of()), + Arguments.of(GraphProjectFromStoreConfig.NODE_PROJECTION_KEY, ImmutableNodeProjections.of()), Arguments.of(GraphProjectFromStoreConfig.RELATIONSHIP_PROJECTION_KEY, ImmutableRelationshipProjections.of()), - Arguments.of(GraphProjectFromStoreConfig.NODE_PROPERTIES_KEY, NodeProjections.of()) + Arguments.of(GraphProjectFromStoreConfig.NODE_PROPERTIES_KEY, ImmutableNodeProjections.of()) ); } diff --git a/core/src/test/java/org/neo4j/gds/config/GraphProjectFromStoreConfigTest.java b/core/src/test/java/org/neo4j/gds/config/GraphProjectFromStoreConfigTest.java index a1b14db1a80..443148ee0f6 100644 --- a/core/src/test/java/org/neo4j/gds/config/GraphProjectFromStoreConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/GraphProjectFromStoreConfigTest.java @@ -25,6 +25,7 @@ import org.neo4j.gds.NodeProjection; import org.neo4j.gds.NodeProjections; import org.neo4j.gds.Orientation; +import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipProjections; @@ -32,7 +33,6 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.core.Aggregation; -import java.util.Collections; import java.util.Set; import static org.hamcrest.MatcherAssert.assertThat; @@ -46,14 +46,18 @@ class GraphProjectFromStoreConfigTest { @Test void testThrowOnOverlappingNodeProperties() { - PropertyMappings propertyMappings = PropertyMappings.builder() - .addMapping("duplicate", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); - - NodeProjections nodeProjections = NodeProjections.create(Collections.singletonMap( - NodeLabel.of("A"), NodeProjection.of("A", propertyMappings) + PropertyMappings propertyMappings = PropertyMappings.of(PropertyMapping.of( + "duplicate", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE )); + var nodeProjections = NodeProjections.single( + NodeLabel.of("A"), + NodeProjection.builder().label("A").properties(propertyMappings).build() + ); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> ImmutableGraphProjectFromStoreConfig.builder() .graphName("graph") @@ -68,9 +72,12 @@ void testThrowOnOverlappingNodeProperties() { @Test void testThrowOnOverlappingRelProperties() { - PropertyMappings propertyMappings = PropertyMappings.builder() - .addMapping("duplicate", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); + var propertyMappings = PropertyMappings.of(PropertyMapping.of( + "duplicate", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE + )); RelationshipProjections relProjections = ImmutableRelationshipProjections.single( RelationshipType.of("A"), @@ -95,18 +102,26 @@ void testThrowOnOverlappingRelProperties() { @Test void testMergingOfNodePropertiesAndProjections() { - PropertyMappings propertyMappings1 = PropertyMappings.builder() - .addMapping("foo", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); - - PropertyMappings propertyMappings2 = PropertyMappings.builder() - .addMapping("bar", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); + var propertyMappings1 = PropertyMappings.of(PropertyMapping.of( + "foo", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE + ) + ); - NodeProjections nodeProjections = NodeProjections.create(Collections.singletonMap( - NodeLabel.of("A"), NodeProjection.of("A", propertyMappings2) + var propertyMappings2 = PropertyMappings.of(PropertyMapping.of( + "bar", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE )); + var nodeProjections = NodeProjections.single( + NodeLabel.of("A"), + NodeProjection.builder().label("A").properties(propertyMappings2).build() + ); + GraphProjectFromStoreConfig graphProjectConfig = ImmutableGraphProjectFromStoreConfig.builder() .graphName("graph") .relationshipProjections(RelationshipProjections.ALL) @@ -122,13 +137,14 @@ void testMergingOfNodePropertiesAndProjections() { @Test void testMergingOfRelationshipPropertiesAndProjections() { - PropertyMappings propertyMappings1 = PropertyMappings.builder() - .addMapping("foo", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); + var propertyMappings1 = PropertyMappings.of(PropertyMapping.of("foo", "foo", DefaultValue.of(0.0), Aggregation.NONE)); - PropertyMappings propertyMappings2 = PropertyMappings.builder() - .addMapping("bar", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); + var propertyMappings2 = PropertyMappings.of(PropertyMapping.of( + "bar", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE + )); RelationshipProjections relProjections = ImmutableRelationshipProjections.single( RelationshipType.of("A"), diff --git a/core/src/test/java/org/neo4j/gds/config/GraphStreamRelationshipsConfigTest.java b/core/src/test/java/org/neo4j/gds/config/GraphStreamRelationshipsConfigTest.java new file mode 100644 index 00000000000..25dab8ff9a8 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/config/GraphStreamRelationshipsConfigTest.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.config; + +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class GraphStreamRelationshipsConfigTest { + + @Test + void validateConcurrency() { + var builder = GraphStreamRelationshipsConfigImpl.builder() + .graphName(Optional.of("g")) + .relationshipTypes(List.of("*")) + .concurrency(42); + + assertThatThrownBy(builder::build).hasMessageContaining("Community users cannot exceed concurrency"); + } + +} diff --git a/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java b/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java index 42de1d59d32..d0de4f6b2eb 100644 --- a/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java @@ -24,11 +24,12 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.loading.GraphStoreBuilder; +import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; -import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import java.util.List; @@ -44,11 +45,16 @@ void validateGraphStoreCapabilities(boolean isBackedByDatabase) { var config = CypherMapWrapper.empty(); var testConfig = new TestWriteConfigImpl(config); + var nodes = ImmutableNodes.builder() + .idMap(new DirectIdMap(0)) + .schema(MutableNodeSchema.empty()) + .build(); + var testGraphStore = new GraphStoreBuilder() .databaseId(DatabaseId.from("neo4j")) .capabilities(ImmutableStaticCapabilities.of(isBackedByDatabase)) - .schema(GraphSchema.empty()) - .nodes(Nodes.of(new DirectIdMap(0))) + .schema(GraphSchema.mutable()) + .nodes(nodes) .relationshipImportResult(RelationshipImportResult.of(Map.of())) .concurrency(1) .build(); diff --git a/core/src/test/java/org/neo4j/gds/core/GraphLoaderMultipleRelTypesAndPropertiesTest.java b/core/src/test/java/org/neo4j/gds/core/GraphLoaderMultipleRelTypesAndPropertiesTest.java index 302452f96ce..c623c4e6656 100644 --- a/core/src/test/java/org/neo4j/gds/core/GraphLoaderMultipleRelTypesAndPropertiesTest.java +++ b/core/src/test/java/org/neo4j/gds/core/GraphLoaderMultipleRelTypesAndPropertiesTest.java @@ -29,7 +29,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.StoreLoaderBuilder; @@ -89,25 +88,14 @@ void setup() { @Test void nodeProjectionsWithExclusiveProperties() { GraphStore graphStore = new StoreLoaderBuilder() - .putNodeProjectionsWithIdentifier( - "N1", - NodeProjection.of( - "Node1", - PropertyMappings.builder().addMapping(PropertyMapping.of("prop1", 0.0D)).build() - ) - ).putNodeProjectionsWithIdentifier( - "N2", - NodeProjection.of( - "Node1", - PropertyMappings.of() - ) - ).putNodeProjectionsWithIdentifier( - "N3", - NodeProjection.of( - "Node2", - PropertyMappings.builder().addMapping(PropertyMapping.of("prop2", 1.0D)).build() - ) - ).graphName("myGraph") + .putNodeProjectionsWithIdentifier("N1", + NodeProjection.builder().label("Node1").addProperty(PropertyMapping.of("prop1", 0.0D)).build() + ) + .putNodeProjectionsWithIdentifier("N2", NodeProjection.of("Node1")) + .putNodeProjectionsWithIdentifier("N3", + NodeProjection.builder().label("Node2").addProperty(PropertyMapping.of("prop2", 1.0D)).build() + ) + .graphName("myGraph") .databaseService(db) .build() .graphStore(); @@ -134,19 +122,14 @@ void nodeProjectionsWithAndWithoutLabel() { GraphStore graphStore = new StoreLoaderBuilder() .putNodeProjectionsWithIdentifier( allIdentifier.name(), - NodeProjection.of( - "*", - PropertyMappings.builder() - .addMapping(PropertyMapping.of("prop1", 42.0D)) - .addMapping(PropertyMapping.of("prop2", 8.0D)) - .build() - ) + NodeProjection + .builder() + .label("*") + .addProperties(PropertyMapping.of("prop1", 42.0D), PropertyMapping.of("prop2", 8.0D)) + .build() ).putNodeProjectionsWithIdentifier( node2Identifier.name(), - NodeProjection.of( - "Node2", - PropertyMappings.builder().addMapping(PropertyMapping.of("prop2", 8.0D)).build() - ) + NodeProjection.builder().label("Node2").addProperty(PropertyMapping.of("prop2", 8.0D)).build() ).graphName("myGraph") .databaseService(db) .build() diff --git a/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java b/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java index 27e7b90c565..5caefb17c78 100644 --- a/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java @@ -47,6 +47,8 @@ import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -212,12 +214,13 @@ public void shouldTrackProgressWithNativeLoadingUsingIndex() { @Test void shouldLogProgressWithCypherLoading() { + var progressRegex = Pattern.compile("(\\d+)%$"); var log = Neo4jProxy.testLog(); new CypherLoaderBuilder() .databaseService(db) .graphName("graph") .nodeQuery("MATCH (n) RETURN id(n) AS id, coalesce(n.prop1, 42) AS prop1") - .relationshipQuery("MATCH (n)-[:REL1|REL2]->(m) RETURN id(n) AS source, id(m) AS target") + .relationshipQuery("MATCH (n)-[:REL1|REL2]-(m) RETURN id(n) AS source, id(m) AS target") .log(log) .build() .graph(); @@ -229,15 +232,20 @@ void shouldLogProgressWithCypherLoading() { "Loading :: Nodes 33%", "Loading :: Nodes 66%", "Loading :: Nodes 100%", - "Loading :: Nodes :: Start", "Loading :: Nodes :: Finished", "Loading :: Relationships :: Start", - "Loading :: Relationships 25%", - "Loading :: Relationships 50%", - "Loading :: Relationships 75%", + "Loading :: Relationships 100%", "Loading :: Relationships :: Finished", "Loading :: Finished" - ); + ) + .noneMatch(message -> { + Matcher matcher = progressRegex.matcher(message); + if (matcher.find()) { + int progress = Integer.parseInt(matcher.group(1)); + return progress > 100; + } + return false; + }); assertThat(log.getMessages(TestLog.DEBUG)).isEmpty(); } diff --git a/core/src/test/java/org/neo4j/gds/core/huge/TransientCsrListTest.java b/core/src/test/java/org/neo4j/gds/core/huge/TransientCsrListTest.java index 2a742ec472b..4d89854a8f7 100644 --- a/core/src/test/java/org/neo4j/gds/core/huge/TransientCsrListTest.java +++ b/core/src/test/java/org/neo4j/gds/core/huge/TransientCsrListTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.AdjacencyCursor; import org.neo4j.gds.api.IdMap; @@ -202,6 +203,7 @@ void shouldWorkWithVeryDenseNodes(TestMethodRunner runner, long firstDegree, lon var relsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(nodes.idMap()) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .build(); @@ -254,7 +256,6 @@ void shouldWorkWithVeryDenseNodes(TestMethodRunner runner, long firstDegree, lon }); } - static AdjacencyCursor adjacencyCursorFromTargets(long[] targets) { long sourceNodeId = targets[0]; NodesBuilder nodesBuilder = GraphFactory.initNodesBuilder() @@ -268,6 +269,7 @@ static AdjacencyCursor adjacencyCursorFromTargets(long[] targets) { RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(RelationshipType.of("REL")) .concurrency(1) .executorService(Pools.DEFAULT) .build(); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java index 94e088a1bc1..6aff8ee9b32 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java @@ -68,12 +68,13 @@ void addRelationshipTypeUndirected() { RelationshipsBuilder relBuilder = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .build(); relBuilder.add(gdlFactory.nodeId("a"), gdlFactory.nodeId("d")); - graphStore.addRelationshipType(RelationshipType.of("NEW"), relBuilder.build()); + graphStore.addRelationshipType(relBuilder.build()); assertThat(graphStore.relationshipCount()).isEqualTo(6); assertThat(graphStore.schema().relationshipSchema().isUndirected()).isEqualTo(true); @@ -95,12 +96,13 @@ void addRelationshipTypeDirected() { RelationshipsBuilder relBuilder = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(RelationshipType.of("NEW")) .orientation(Orientation.NATURAL) .build(); relBuilder.add(gdlFactory.nodeId("a"), gdlFactory.nodeId("d")); - graphStore.addRelationshipType(RelationshipType.of("NEW"), relBuilder.build()); + graphStore.addRelationshipType(relBuilder.build()); assertThat(graphStore.relationshipCount()).isEqualTo(3); assertThat(graphStore.schema().relationshipSchema().isUndirected()).isEqualTo(false); @@ -166,12 +168,13 @@ void addRelationshipTypeMixed(Orientation baseOrientation, int baseRelCount, Ori RelationshipsBuilder relBuilder = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(RelationshipType.of("REL")) .orientation(addedOrientation) .build(); relBuilder.add(gdlFactory.nodeId("a"), gdlFactory.nodeId("d")); - graphStore.addRelationshipType(RelationshipType.of("NEW"), relBuilder.build()); + graphStore.addRelationshipType(relBuilder.build()); assertThat(graphStore.relationshipCount()).isEqualTo(totalRelCount); assertThat(graphStore.schema().relationshipSchema().isUndirected()).isEqualTo(false); @@ -250,7 +253,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valueCount() { return 4; } }); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java index e7fac8785d8..84129d978dd 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java @@ -56,7 +56,6 @@ void fromGraph() { var convertedGraphStore = CSRGraphStoreUtil.createFromGraph( DatabaseId.from("dummy"), (HugeGraph) graph, - "REL1", Optional.of("prop1"), 1 ); @@ -65,6 +64,28 @@ void fromGraph() { assertGraphEquals(graphStore.getUnion(), convertedGraphStore.getUnion()); } + @Test + void fromGraphWithNoRelationships() { + String GDL = + " CREATE" + + " (a:A { foo: 42, bar: 1337 })" + + ", (b:A { foo: 84, bar: 1234 })" + + ", (c:B { foo: 23 })"; + + var gdlGraphStore = TestSupport.graphStoreFromGDL(GDL); + var gdlGraph = gdlGraphStore.getUnion(); + + var convertedGraphStore = CSRGraphStoreUtil.createFromGraph( + DatabaseId.from("dummy"), + (HugeGraph) gdlGraph, + Optional.empty(), + 1 + ); + + assertThat(convertedGraphStore.schema()).isEqualTo(gdlGraphStore.schema()); + assertGraphEquals(gdlGraphStore.getUnion(), convertedGraphStore.getUnion()); + } + @Test void shouldValidateRelationshipPropertyKey() { var graph = TestSupport.fromGdl("()-[:REL]->()"); @@ -73,7 +94,6 @@ void shouldValidateRelationshipPropertyKey() { CSRGraphStoreUtil.createFromGraph( DatabaseId.from("dummy"), (HugeGraph) graph.innerGraph(), - "REL", Optional.of("prop1"), 1 ); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java index 6ec6459bfd7..8ecc69f74ea 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java @@ -532,8 +532,8 @@ private static Stream memoryEstimationVariants() { "Node properties", "MATCH (n) RETURN id(n) as id, n.id as idProp", "MATCH (n)-[r]->(m) RETURN id(n) AS source, id(m) AS target", - 1300728, - 1300728 + 1300720, + 1300720 ), Arguments.of( diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CypherQueryEstimatorTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CypherQueryEstimatorTest.java new file mode 100644 index 00000000000..0dcc4f1af35 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/CypherQueryEstimatorTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseTest; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.gds.transaction.TransactionContext; + +import static org.assertj.core.api.Assertions.assertThat; + +class CypherQueryEstimatorTest extends BaseTest { + + @Neo4jGraph + private static final String DB_CYPHER = + "CREATE" + + " (a1:A {property: 33, a: 33})" + + ", (a2:A {property: 33, a: 33})" + + ", (b:B {property: 42, b: 42})" + + ", (a1)-[:T1 {property1: 42, property2: 1337}]->(b)" + + ", (a2)-[:T2 {property1: 43}]->(b)" + + ", (a2)-[:T2 {property1: 43}]->(a1)"; + + @Test + void estimateNodes() { + GraphDatabaseApiProxy.runInTransaction(db, tx -> { + CypherQueryEstimator estimator = new CypherQueryEstimator(TransactionContext.of(db, tx)); + + var estimation = estimator.getNodeEstimation( + "MATCH (n) RETURN id(n) AS id, labels(n) AS labels, n.property AS score"); + + // EXPLAIN seems to overestimate the nodeCount here + assertThat(estimation).isEqualTo(ImmutableEstimationResult.of(10, 1)); + }); + } + + @Test + void estimateRelationships() { + GraphDatabaseApiProxy.runInTransaction(db, tx -> { + CypherQueryEstimator estimator = new CypherQueryEstimator(TransactionContext.of(db, tx)); + + var estimation = estimator.getRelationshipEstimation( + "MATCH (n)-[r]-(m) RETURN id(n) AS source, m AS target, r.property1 AS score, type(r) AS type"); + + assertThat(estimation).isEqualTo(ImmutableEstimationResult.of(6, 1)); + }); + } + + +} diff --git a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java index e4f6fc64510..18e9e7ad556 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.loading; +import org.apache.commons.lang3.mutable.MutableInt; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.DatabaseId; @@ -414,6 +415,28 @@ void multipleDatabaseIds() { assertFalse(GraphStoreCatalog.exists(USER_NAME, databaseId1, "graph0")); } + @Test + void callListeners() { + var listenerCalls = new MutableInt(0); + + var listener = new GraphStoreCatalogListener() { + @Override + public void onProject(String user, String database, String graphName) { + listenerCalls.increment(); + } + }; + + GraphStoreCatalog.registerListener(listener); + assertThat(listenerCalls.intValue()).isZero(); + + GraphStoreCatalog.set(CONFIG, graphStore); + assertThat(listenerCalls.intValue()).isEqualTo(1); + + GraphStoreCatalog.unregisterListener(listener); + GraphStoreCatalog.set(GraphProjectFromStoreConfig.emptyWithName("bob", GRAPH_NAME), graphStore); + assertThat(listenerCalls.intValue()).isEqualTo(1); + } + @Test void shouldThrowOnMissingGraph() { var dummyDatabaseId = DatabaseId.from("mydatabase"); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java index c5c46c427d2..89714b13885 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java @@ -171,9 +171,10 @@ void testModificationDate() throws InterruptedException { // add relationships Thread.sleep(42); + RelationshipType bar = RelationshipType.of("BAR"); graphStore.addRelationshipType( - RelationshipType.of("BAR"), SingleTypeRelationships.of( + bar, ImmutableTopology.of(CompressedAdjacencyList.EMPTY, 0, false), Direction.DIRECTED, Optional.empty(), @@ -279,18 +280,18 @@ void nodeOnlyGraph() { private static List nodeProjections() { NodeProjection aMapping = NodeProjection.builder() .label("A") - .properties(PropertyMappings.of(Arrays.asList( + .addProperties( PropertyMapping.of("nodeProperty", -1D), PropertyMapping.of("a", -1D) - ))) + ) .build(); NodeProjection bMapping = NodeProjection.builder() .label("B") - .properties(PropertyMappings.of(Arrays.asList( + .addProperties( PropertyMapping.of("nodeProperty", -1D), PropertyMapping.of("b", -1D) - ))) + ) .build(); return Arrays.asList(aMapping, bMapping); @@ -302,32 +303,24 @@ private static List relationshipProjections() { .type("T1") .orientation(Orientation.NATURAL) .aggregation(Aggregation.NONE) - .properties( - PropertyMappings.builder() - .addMapping("property1", "property1", DefaultValue.of(42D), Aggregation.NONE) - .addMapping("property2", "property2", DefaultValue.of(1337D), Aggregation.NONE) - .build() + .addProperties( + PropertyMapping.of("property1", "property1", DefaultValue.of(42D), Aggregation.NONE), + PropertyMapping.of("property2", "property2", DefaultValue.of(1337D), Aggregation.NONE) ).build(); RelationshipProjection t2Mapping = RelationshipProjection.builder() .type("T2") .orientation(Orientation.NATURAL) .aggregation(Aggregation.NONE) - .properties( - PropertyMappings.builder() - .addMapping("property1", "property1", DefaultValue.of(42D), Aggregation.NONE) - .build() - ).build(); + .addProperty(PropertyMapping.of("property1", "property1", DefaultValue.of(42D), Aggregation.NONE)) + .build(); RelationshipProjection t3Mapping = RelationshipProjection.builder() .type("T3") .orientation(Orientation.NATURAL) .aggregation(Aggregation.NONE) - .properties( - PropertyMappings.builder() - .addMapping("property2", "property2", DefaultValue.of(42D), Aggregation.NONE) - .build() - ).build(); + .addProperty(PropertyMapping.of("property2", "property2", DefaultValue.of(42D), Aggregation.NONE)) + .build(); return Arrays.asList(t1Mapping, t2Mapping, t3Mapping); } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/HugeGraphLoadingTest.java b/core/src/test/java/org/neo4j/gds/core/loading/HugeGraphLoadingTest.java index 553c6d29cd1..0e2af30bf08 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/HugeGraphLoadingTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/HugeGraphLoadingTest.java @@ -105,10 +105,10 @@ private void testPropertyLoading(int maxArrayLengthShift) { .graph(); NodePropertyValues nodePropertyValues = graph.nodeProperties("bar"); - long propertyCountDiff = nodeCount - nodePropertyValues.size(); + long propertyCountDiff = nodeCount - nodePropertyValues.nodeCount(); String errorMessage = formatWithLocale( "Expected %d properties to be imported. Actually imported %d properties (missing %d properties).", - nodeCount, nodePropertyValues.size(), propertyCountDiff + nodeCount, nodePropertyValues.nodeCount(), propertyCountDiff ); assertEquals(0, propertyCountDiff, errorMessage); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java new file mode 100644 index 00000000000..947f4846c3e --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.core.concurrency.ParallelUtil; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.loading.construction.NodeLabelTokens; +import org.neo4j.gds.core.utils.partition.PartitionUtils; + +import java.util.Collections; +import java.util.Optional; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +class LazyIdMapBuilderTest { + + @Test + void parallelAddDuplicateNodes() { + int concurrency = 8; + long idCount = 100_000; + var rng = new Random(42); + + var lazyIdMapBuilder = new LazyIdMapBuilder(concurrency, false, false); + + var idList = LongStream.rangeClosed(1, idCount) + .flatMap(id -> rng.nextBoolean() ? LongStream.of(id, id) : LongStream.of(id)) + .limit(idCount) + .boxed() + .collect(Collectors.toList()); + + // shuffle to spread duplicates across the whole range + Collections.shuffle(idList); + + var idArray = idList.stream().mapToLong(l -> l).toArray(); + + var tasks = PartitionUtils.rangePartition(concurrency, idCount, partition -> (Runnable) () -> { + int start = (int) partition.startNode(); + int end = (int) (start + partition.nodeCount()); + for (int i = start; i < end; i++) { + long originalId = idArray[i]; + // We potentially insert the same original id from multiple threads. + // This MUST not lead to new intermediate ids generated internally. + lazyIdMapBuilder.addNode(originalId, NodeLabelTokens.empty()); + } + + }, Optional.empty()); + + ParallelUtil.run(tasks, Pools.DEFAULT); + + var highLimitIdMap = lazyIdMapBuilder.build().idMap(); + + Assertions.assertThat(highLimitIdMap.nodeCount()).isLessThan(idCount); + + for (int internalNodeId = 0; internalNodeId < highLimitIdMap.nodeCount(); internalNodeId++) { + Assertions + .assertThat(highLimitIdMap.toOriginalNodeId(internalNodeId)) + .as("Internal node id %s is not mapped", internalNodeId) + .isNotEqualTo(0); + } + } + +} diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NodeLabelIndexTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NodeLabelIndexTest.java index 03fa1be388f..372667ee5c7 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NodeLabelIndexTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NodeLabelIndexTest.java @@ -27,6 +27,9 @@ import org.neo4j.gds.extension.Neo4jGraph; import org.neo4j.gds.extension.Neo4jGraphExtension; +import java.util.ArrayList; +import java.util.List; + import static org.assertj.core.api.Assertions.assertThat; import static org.neo4j.gds.TestSupport.assertGraphEquals; import static org.neo4j.gds.TestSupport.fromGdl; @@ -39,17 +42,26 @@ public class NodeLabelIndexTest extends BaseTest { @Test void shouldLoadWithoutNodeLabelIndex() { - runQueryWithResultConsumer( - "SHOW INDEXES WHERE entityType = 'NODE'", + List nodeIndices = runQuery( + "SHOW INDEXES " + + " YIELD name, entityType " + + " WHERE entityType = 'NODE'" + + " RETURN name", result -> { - assertThat(result.hasNext()).isTrue(); - runQueryWithResultConsumer( - "DROP INDEX " + result.next().get("name"), - innerResult -> assertThat(innerResult.resultAsString()).contains("Indexes removed: 1") - ); + var indices = new ArrayList(); + while (result.hasNext()) { + indices.add((String) result.next().get("name")); + } + return indices; } ); + nodeIndices.forEach(index -> runQuery( + "DROP INDEX " + index, + result -> assertThat(result.resultAsString()).contains("Indexes removed: 1") + )); + + var log = Neo4jProxy.testLog();; var graph = new StoreLoaderBuilder() .databaseService(db) diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java index a1ee3505add..3b0b98116a3 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java @@ -61,7 +61,7 @@ void testEmptyDoubleProperties() { 1 ).build(idMap(nodeCount)); - assertEquals(0L, properties.size()); + assertEquals(nodeCount, properties.nodeCount()); assertEquals(OptionalDouble.empty(), properties.getMaxDoublePropertyValue()); assertEquals(42.0, properties.doubleValue(0)); } @@ -74,7 +74,7 @@ void testEmptyLongProperties() { 1 ).build(idMap(nodeCount)); - assertEquals(0L, properties.size()); + assertEquals(nodeCount, properties.nodeCount()); assertEquals(OptionalLong.empty(), properties.getMaxLongPropertyValue()); assertEquals(42, properties.longValue(0)); } @@ -241,7 +241,7 @@ void hasSize() { b.set(0, Values.of(42.0)); b.set(1, Values.of(21.0)); }); - assertEquals(2, properties.size()); + assertEquals(2, properties.nodeCount()); } @Test @@ -304,7 +304,7 @@ void threadSafety() throws InterruptedException { var expected = i == 1338 ? 0x1p41 : i == 1337 ? 0x1p42 : i % 2 == 0 ? 2.0 : 1.0; assertEquals(expected, properties.doubleValue(i), "" + i); } - assertEquals(nodeSize, properties.size()); + assertEquals(nodeSize, properties.nodeCount()); var maxPropertyValue = properties.getMaxDoublePropertyValue(); assertTrue(maxPropertyValue.isPresent()); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java new file mode 100644 index 00000000000..2de63acfcbc --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -0,0 +1,253 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.nodeproperties.ValueType; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.PropertySchema; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class NodeLabelTokenToPropertyKeysTest { + + @Test + void testPropertySchemasLazy() { + var mapping = NodeLabelTokenToPropertyKeys.lazy(); + + mapping.add(NodeLabelTokens.ofStrings("A"), List.of("foo", "bar")); + mapping.add(NodeLabelTokens.ofStrings("A"), List.of("baz")); + mapping.add(NodeLabelTokens.ofStrings("B"), List.of("baz", "bob")); + mapping.add(NodeLabelTokens.ofStrings("C"), List.of()); + + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bob", PropertySchema.of("bob", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + ); + + assertThat(mapping.propertySchemas(NodeLabel.of("A"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("B"), importPropertySchemas)).isEqualTo(Map.of( + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bob", PropertySchema.of("bob", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("C"), importPropertySchemas)).isEqualTo(Map.of()); + } + + @Test + void testPropertySchemasFixed() { + var nodeSchema = MutableNodeSchema.empty() + .addLabel(NodeLabel.of("A"), Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG), + "bar", PropertySchema.of("bar", ValueType.DOUBLE), + "baz", PropertySchema.of("baz", ValueType.LONG) + )) + .addLabel(NodeLabel.of("B"), Map.of( + "baz", PropertySchema.of("baz", ValueType.LONG), + "bob", PropertySchema.of("bob", ValueType.LONG) + )) + .addLabel(NodeLabel.of("C")); + + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bob", PropertySchema.of("bob", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + ); + + var mapping = NodeLabelTokenToPropertyKeys.fixed(nodeSchema); + + assertThat(mapping.propertySchemas(NodeLabel.of("A"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("B"), importPropertySchemas)).isEqualTo(Map.of( + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bob", PropertySchema.of("bob", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("C"), importPropertySchemas)).isEqualTo(Map.of()); + } + + @Test + void shouldFailForMissingProperties() { + var nodeSchema = MutableNodeSchema.empty() + .addLabel(NodeLabel.of("A"), Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG), + "baz", PropertySchema.of("baz", ValueType.LONG) + )); + + var fixed = NodeLabelTokenToPropertyKeys.fixed(nodeSchema); + + assertThatThrownBy(() -> fixed.propertySchemas( + NodeLabel.of("A"), + Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG) + ) + )) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Missing node properties during import.") + .hasMessageContaining("['baz']"); + } + + @Test + void shouldFailForIncompatibleTypes() { + var nodeSchema = MutableNodeSchema.empty() + .addLabel(NodeLabel.of("A"), Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG), + "baz", PropertySchema.of("baz", ValueType.LONG) + )); + + var fixed = NodeLabelTokenToPropertyKeys.fixed(nodeSchema); + + assertThatThrownBy(() -> fixed.propertySchemas( + NodeLabel.of("A"), + Map.of( + "foo", PropertySchema.of("foo", ValueType.DOUBLE), + "baz", PropertySchema.of("baz", ValueType.LONG) + ) + )) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Incompatible value types between input schema and input data.") + .hasMessageContaining("['foo']"); + } + + @Test + void testDuplicatePropertyKeys() { + var mapping = NodeLabelTokenToPropertyKeys.lazy(); + mapping.add(NodeLabelTokens.ofStrings("A", "B"), List.of("foo", "bar")); + mapping.add(NodeLabelTokens.ofStrings("A"), List.of("foo")); + mapping.add(NodeLabelTokens.ofStrings("B"), List.of("bar")); + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + ); + assertThat(mapping.propertySchemas(NodeLabel.of("A"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("B"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + )); + } + + @Test + void testEmptyNodeLabelTokenResolvesToAllNodes() { + var mapping = NodeLabelTokenToPropertyKeys.lazy(); + mapping.add(NodeLabelTokens.empty(), List.of("foo", "bar")); + + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + ); + + assertThat(mapping.propertySchemas(NodeLabel.ALL_NODES, importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + )); + } + + @Test + void testNodeLabelsLazily() { + var mapping = NodeLabelTokenToPropertyKeys.lazy(); + mapping.add(NodeLabelTokens.empty(), List.of("foo")); + mapping.add(NodeLabelTokens.of("A"), List.of("bar")); + mapping.add(NodeLabelTokens.ofStrings("A", "B"), List.of("baz")); + mapping.add(NodeLabelTokens.ofStrings("B", "C"), List.of("buz")); + + assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder( + NodeLabel.ALL_NODES, + NodeLabel.of("A"), + NodeLabel.of("B"), + NodeLabel.of("C") + ); + } + + @Test + void testNodeLabelsFixed() { + var mapping = NodeLabelTokenToPropertyKeys.fixed(MutableNodeSchema + .empty() + .addLabel(NodeLabel.ALL_NODES) + .addLabel(NodeLabel.of("A")) + .addLabel(NodeLabel.of("B")) + .addLabel(NodeLabel.of("C"))); + + assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder( + NodeLabel.ALL_NODES, + NodeLabel.of("A"), + NodeLabel.of("B"), + NodeLabel.of("C") + ); + } + + @Test + void testMerge() { + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG_ARRAY, DefaultValue.forLongArray(), PropertyState.PERSISTENT) + ); + + var aLabel = NodeLabel.of("A"); + var bLabel = NodeLabel.of("B"); + var cLabel = NodeLabel.of("C"); + + var mapping0 = NodeLabelTokenToPropertyKeys.lazy(); + mapping0.add(NodeLabelTokens.ofNodeLabels(aLabel), List.of("foo")); + mapping0.add(NodeLabelTokens.ofNodeLabels(cLabel), List.of("bar")); + + var mapping1 = NodeLabelTokenToPropertyKeys.lazy(); + mapping1.add(NodeLabelTokens.ofNodeLabels(bLabel), List.of("bar")); + + var mapping2 = NodeLabelTokenToPropertyKeys.lazy(); + mapping2.add(NodeLabelTokens.ofNodeLabels(aLabel, cLabel), List.of("baz")); + + var union = NodeLabelTokenToPropertyKeys.union(mapping0, mapping1, importPropertySchemas); + union = NodeLabelTokenToPropertyKeys.union(union, mapping2, importPropertySchemas); + + assertThat(union.nodeLabels()).containsExactlyInAnyOrder(aLabel, bLabel, cLabel); + + assertThat(union.propertySchemas(aLabel, importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "baz", PropertySchema.of("baz", ValueType.LONG_ARRAY, DefaultValue.forLongArray(), PropertyState.PERSISTENT) + )); + assertThat(union.propertySchemas(bLabel, importPropertySchemas)).isEqualTo(Map.of( + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + )); + assertThat(union.propertySchemas(cLabel, importPropertySchemas)).isEqualTo(Map.of( + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG_ARRAY, DefaultValue.forLongArray(), PropertyState.PERSISTENT) + )); + } +} + diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokensTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokensTest.java new file mode 100644 index 00000000000..155cfa24b69 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokensTest.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.NodeLabel; + +import static org.assertj.core.api.Assertions.assertThat; + +class NodeLabelTokensTest { + + @Test + void testStream() { + var aLabel = NodeLabel.of("A"); + var bLabel = NodeLabel.of("B"); + var cLabel = NodeLabel.of("C"); + + assertThat(NodeLabelTokens.empty().nodeLabels()).isEmpty(); + assertThat(NodeLabelTokens.ofNodeLabel(aLabel).nodeLabels()).contains(aLabel); + assertThat( + NodeLabelTokens.ofNodeLabels(aLabel, bLabel, cLabel).nodeLabels() + ).contains(aLabel, bLabel, cLabel); + assertThat( + NodeLabelTokens.ofStrings(aLabel.name(), bLabel.name(), cLabel.name()).nodeLabels() + ).contains(aLabel, bLabel, cLabel); + } + +} diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeObjectArrayTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeObjectArrayTest.java index 3bf8ca7b870..88013671d69 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeObjectArrayTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeObjectArrayTest.java @@ -53,7 +53,7 @@ void shouldReturnNodePropertiesForFloatArrayValues(HugeObjectArray floa assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(FloatArrayNodePropertyValues.class)) .describedAs("float properties must return the same size and elements as the underlying array") - .returns(floats.size(), FloatArrayNodePropertyValues::size) + .returns(floats.size(), FloatArrayNodePropertyValues::nodeCount) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.floatArrayValue(nodeId)) @@ -75,7 +75,7 @@ void shouldReturnNodePropertiesForDoubleArrayValues(HugeObjectArray do assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(DoubleArrayNodePropertyValues.class)) .describedAs("double properties must return the same size and elements as the underlying array") - .returns(doubles.size(), DoubleArrayNodePropertyValues::size) + .returns(doubles.size(), DoubleArrayNodePropertyValues::nodeCount) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.doubleArrayValue(nodeId)) @@ -104,7 +104,7 @@ void shouldReturnNodePropertiesForLongArrayValues(HugeObjectArray longs) assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(LongArrayNodePropertyValues.class)) .describedAs("long properties must return the same size and elements as the underlying array") - .returns(longs.size(), LongArrayNodePropertyValues::size) + .returns(longs.size(), LongArrayNodePropertyValues::nodeCount) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.longArrayValue(nodeId)) diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java index ecebd86ad05..2393f1a7a27 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java @@ -110,6 +110,16 @@ void testContains(@ForAll("ids") long[] originalIds) { .forEach(id -> assertThat(map.contains(id)).isFalse()); } + @Property + void testAddNode(@ForAll("ids") long[] originalIds) { + var builder = builder(1); + for (int i = 0; i < originalIds.length; i++) { + long originalId = originalIds[i]; + long mappedId = builder.addNode(originalId); + assertThat(mappedId).isEqualTo(i); + } + } + @Property void testMaxOriginalId(@ForAll("ids") long[] originalIds) { var builder = builder(1); @@ -189,6 +199,8 @@ void testAddingMultipleNodesInParallel(@ForAll("fixedSizeIds") long[] originalId abstract TestBuilder builder(int concurrency); interface TestBuilder { + long addNode(long nodeId); + void addNodes(long... nodeIds); ShardedLongLongMap build(); @@ -196,6 +208,16 @@ interface TestBuilder { static class DefaultBuilderTest extends ShardedLongLongMapTest { + @Property + void testAddNodeWithDuplicates(@ForAll("ids") long[] originalIds) { + var builder = builder(1); + for (long originalId : originalIds) { + long mappedId = builder.addNode(originalId); + long duplicateMappedId = builder.addNode(originalId); + assertThat(duplicateMappedId).isEqualTo(-mappedId - 1); + } + } + @Override TestBuilder builder(int concurrency) { return new DefaultBuilder(concurrency); @@ -208,6 +230,11 @@ private static final class DefaultBuilder implements TestBuilder { this.inner = ShardedLongLongMap.builder(concurrency); } + @Override + public long addNode(long nodeId) { + return inner.addNode(nodeId); + } + @Override public void addNodes(long... nodeIds) { for (long nodeId : nodeIds) { @@ -248,6 +275,12 @@ private static final class BatchedBuilder implements TestBuilder { inner = ShardedLongLongMap.batchedBuilder(concurrency); } + @Override + public long addNode(long nodeId) { + var batch = inner.prepareBatch(1); + return batch.addNode(nodeId); + } + @Override public void addNodes(long... nodeIds) { var batch = inner.prepareBatch(nodeIds.length); diff --git a/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java b/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java index 3bec5f88e5b..3c6a4ec5547 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.utils.progress.tasks; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; @@ -27,15 +28,19 @@ import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistry; import org.neo4j.gds.core.utils.progress.TaskStore; +import org.neo4j.gds.utils.GdsFeatureToggles; import org.neo4j.logging.Log; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.neo4j.gds.assertj.Extractors.removingThreadId; +import static org.neo4j.gds.compat.TestLog.WARN; -public class TaskProgressTrackerTest { +class TaskProgressTrackerTest { @Test void shouldStepThroughSubtasks() { @@ -72,14 +77,35 @@ void shouldStepThroughSubtasks() { } @Test - void shouldThrowIfEndMoreTasksThanStarted() { - var task = Tasks.leaf("leaf"); - TaskProgressTracker progressTracker = progressTracker(task); - progressTracker.beginSubTask(); - progressTracker.endSubTask(); - assertThatThrownBy(progressTracker::endSubTask) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Tried to log progress, but there are no running tasks being tracked"); + void shouldThrowIfEndMoreTasksThanStartedAndFeatureToggleIsEnabled() { + GdsFeatureToggles.THROW_WHEN_USING_PROGRESS_TRACKER_WITHOUT_TASKS.enableAndRun(() -> { + var task = Tasks.leaf("leaf"); + TaskProgressTracker progressTracker = progressTracker(task); + progressTracker.beginSubTask(); + progressTracker.endSubTask(); + assertThatThrownBy(progressTracker::endSubTask) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Tried to log progress, but there are no running tasks being tracked"); + }); + } + + @Test + void shouldNotThrowIfEndMoreTasksThanStarted() { + GdsFeatureToggles.THROW_WHEN_USING_PROGRESS_TRACKER_WITHOUT_TASKS.disableAndRun(() -> { + var task = Tasks.leaf("leaf"); + var log = Neo4jProxy.testLog(); + var progressTracker = new TaskProgressTracker(task, log, 1, EmptyTaskRegistryFactory.INSTANCE); + progressTracker.beginSubTask(); + progressTracker.endSubTask(); + assertThatNoException() + .as("When `THROW_WHEN_USING_PROGRESS_TRACKER_WITHOUT_TASKS` is disabled (default state) we should not throw an exception.") + .isThrownBy(progressTracker::endSubTask); + + Assertions + .assertThat(log.getMessages(WARN)) + .extracting(removingThreadId()) + .containsExactly("leaf :: Tried to log progress, but there are no running tasks being tracked"); + }); } @Test diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java index 1125bc0ade1..c348737a88e 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java @@ -92,7 +92,7 @@ public CompatUserAggregator create(Context ctx) throws ProcedureException { .apply(ctx); var username = procedures.lookupComponentProvider(Username.class, true).apply(ctx); - var runsOnCompositeDatabase = DatabaseTopologyHelper.isCompositeDatabase(databaseService); + var runsOnCompositeDatabase = Neo4jProxy.isCompositeDatabase(databaseService); return new GraphAggregator( DatabaseId.of(databaseService), diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index 206d27bda9f..9f0d8fabf07 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -21,22 +21,17 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.DefaultValue; -import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; -import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.ImmutableMutableGraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.compat.CompatUserAggregator; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.ConfigKeyValidation; import org.neo4j.gds.core.compress.AdjacencyCompressor; -import org.neo4j.gds.core.loading.CSRGraphStoreUtil; import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.loading.ImmutableNodes; @@ -45,6 +40,7 @@ import org.neo4j.gds.core.loading.ReadHelper; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; +import org.neo4j.gds.core.loading.construction.ImmutablePropertyConfig; import org.neo4j.gds.core.loading.construction.NodeLabelToken; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; import org.neo4j.gds.core.loading.construction.PropertyValues; @@ -61,8 +57,6 @@ import org.neo4j.values.virtual.MapValue; import org.neo4j.values.virtual.MapValueBuilder; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -150,11 +144,16 @@ void projectNextRelationship( AnyValue relationshipConfig, AnyValue config ) { + this.configValidator.validateConfigs(nodesConfig, relationshipConfig); + + var data = initGraphData(graphName, config); + @Nullable MapValue sourceNodePropertyValues = null; @Nullable MapValue targetNodePropertyValues = null; NodeLabelToken sourceNodeLabels = NodeLabelTokens.missing(); NodeLabelToken targetNodeLabels = NodeLabelTokens.missing(); + this.configValidator.validateConfigs(nodesConfig, relationshipConfig); if (nodesConfig instanceof MapValue) { sourceNodePropertyValues = GraphImporter.propertiesConfig("sourceNodeProperties", (MapValue) nodesConfig); sourceNodeLabels = labelsConfig("sourceNodeLabels", (MapValue) nodesConfig); @@ -166,20 +165,8 @@ void projectNextRelationship( ); targetNodeLabels = labelsConfig("targetNodeLabels", (MapValue) nodesConfig); } - - this.configValidator.validateNodesConfig((MapValue) nodesConfig); } - var data = initGraphData( - graphName, - config, - targetNodePropertyValues, - sourceNodePropertyValues, - targetNodeLabels, - sourceNodeLabels, - relationshipConfig - ); - data.update( sourceNode, targetNode, @@ -187,20 +174,11 @@ void projectNextRelationship( targetNodePropertyValues, sourceNodeLabels, targetNodeLabels, - relationshipConfig, - this.configValidator + relationshipConfig ); } - private GraphImporter initGraphData( - TextValue graphName, - AnyValue config, - @Nullable MapValue sourceNodePropertyValues, - @Nullable MapValue targetNodePropertyValues, - NodeLabelToken sourceNodeLabels, - NodeLabelToken targetNodeLabels, - AnyValue relationshipConfig - ) { + private GraphImporter initGraphData(TextValue graphName, AnyValue config) { var data = this.importer; if (data != null) { return data; @@ -215,13 +193,7 @@ private GraphImporter initGraphData( this.username, this.databaseId, config, - sourceNodePropertyValues, - targetNodePropertyValues, - sourceNodeLabels, - targetNodeLabels, - relationshipConfig, - this.canWriteToDatabase, - this.lock + this.canWriteToDatabase ); } return data; @@ -307,52 +279,54 @@ private static final class ConfigValidator { "relationshipType" ); - private final AtomicBoolean validateNodes = new AtomicBoolean(true); - private final AtomicBoolean validateRelationships = new AtomicBoolean(true); - void validateNodesConfig(MapValue nodesConfig) { - if (this.validateNodes.getAndSet(false)) { - ConfigKeyValidation.requireOnlyKeysFrom(NODES_CONFIG_KEYS, nodesConfig.keySet()); - } - } - - void validateRelationshipsConfig(MapValue relationshipConfig) { - if (this.validateRelationships.getAndSet(false)) { - ConfigKeyValidation.requireOnlyKeysFrom(RELATIONSHIPS_CONFIG_KEYS, relationshipConfig.keySet()); + private final AtomicBoolean validate = new AtomicBoolean(true); + + void validateConfigs(AnyValue nodesConfig, AnyValue relationshipConfig) { + if (nodesConfig instanceof MapValue || relationshipConfig instanceof MapValue) { + if (this.validate.get()) { + if (this.validate.getAndSet(false)) { + if (nodesConfig instanceof MapValue) { + ConfigKeyValidation.requireOnlyKeysFrom( + NODES_CONFIG_KEYS, + ((MapValue) nodesConfig).keySet() + ); + } + if (relationshipConfig instanceof MapValue) { + ConfigKeyValidation.requireOnlyKeysFrom( + RELATIONSHIPS_CONFIG_KEYS, + ((MapValue) relationshipConfig).keySet() + ); + } + } + } } } } // Does the actual importing work once we can initialize it with the first row - @SuppressWarnings("CodeBlock2Expr") private static final class GraphImporter { private final String graphName; private final GraphProjectFromCypherAggregationConfig config; private final LazyIdMapBuilder idMapBuilder; - private final @Nullable List relationshipPropertySchemas; private final boolean canWriteToDatabase; private final ExtractNodeId extractNodeId; - private final Lock lock; private final Map relImporters; - private final ImmutableGraphSchema.Builder graphSchemaBuilder; + private final ImmutableMutableGraphSchema.Builder graphSchemaBuilder; private GraphImporter( String graphName, GraphProjectFromCypherAggregationConfig config, LazyIdMapBuilder idMapBuilder, - @Nullable List relationshipPropertySchemas, - boolean canWriteToDatabase, - Lock lock + boolean canWriteToDatabase ) { this.graphName = graphName; this.config = config; this.idMapBuilder = idMapBuilder; - this.relationshipPropertySchemas = relationshipPropertySchemas; this.canWriteToDatabase = canWriteToDatabase; - this.lock = lock; this.relImporters = new ConcurrentHashMap<>(); - this.graphSchemaBuilder = ImmutableGraphSchema.builder(); + this.graphSchemaBuilder = MutableGraphSchema.builder(); this.extractNodeId = new ExtractNodeId(); } @@ -361,13 +335,7 @@ static GraphImporter of( String username, DatabaseId databaseId, AnyValue configMap, - @Nullable MapValue sourceNodePropertyValues, - @Nullable MapValue targetNodePropertyValues, - NodeLabelToken sourceNodeLabels, - NodeLabelToken targetNodeLabels, - AnyValue relationshipConfig, - boolean canWriteToDatabase, - Lock lock + boolean canWriteToDatabase ) { var graphName = graphNameValue.stringValue(); @@ -379,24 +347,9 @@ static GraphImporter of( (configMap instanceof MapValue) ? (MapValue) configMap : MapValue.EMPTY ); - var idMapBuilder = idMapBuilder( - sourceNodeLabels, - sourceNodePropertyValues, - targetNodeLabels, - targetNodePropertyValues, - config.readConcurrency() - ); - - var relationshipPropertySchemas = relationshipPropertySchemas(relationshipConfig); + var idMapBuilder = idMapBuilder(config.readConcurrency()); - return new GraphImporter( - graphName, - config, - idMapBuilder, - relationshipPropertySchemas, - canWriteToDatabase, - lock - ); + return new GraphImporter(graphName, config, idMapBuilder, canWriteToDatabase); } private static void validateGraphName(String graphName, String username, DatabaseId databaseId) { @@ -405,46 +358,8 @@ private static void validateGraphName(String graphName, String username, Databas } } - private static LazyIdMapBuilder idMapBuilder( - NodeLabelToken sourceNodeLabels, - @Nullable MapValue sourceNodeProperties, - NodeLabelToken targetNodeLabels, - @Nullable MapValue targetNodeProperties, - int readConcurrency - ) { - boolean hasLabelInformation = !(sourceNodeLabels.isMissing() && targetNodeLabels.isMissing()); - boolean hasProperties = !(sourceNodeProperties == null && targetNodeProperties == null); - return new LazyIdMapBuilder(readConcurrency, hasLabelInformation, hasProperties); - } - - private static @Nullable List relationshipPropertySchemas(AnyValue relationshipConfigValue) { - if (!(relationshipConfigValue instanceof MapValue)) { - return null; - } - - //noinspection PatternVariableCanBeUsed - var relationshipConfig = (MapValue) relationshipConfigValue; - - var relationshipPropertySchemas = new ArrayList(); - - // We need to do this before extracting the `relationshipProperties`, because - // we remove the original entry from the map during converting; also we remove null keys - // so we could not create a schema entry for properties that are absent on the current relationship - var relationshipPropertyKeys = relationshipConfig.get("properties"); - if (relationshipPropertyKeys instanceof MapValue) { - for (var propertyKey : ((MapValue) relationshipPropertyKeys).keySet()) { - relationshipPropertySchemas.add(RelationshipPropertySchema.of( - propertyKey, - ValueType.DOUBLE - )); - } - } - - if (relationshipPropertySchemas.isEmpty()) { - return null; - } - - return relationshipPropertySchemas; + private static LazyIdMapBuilder idMapBuilder(int readConcurrency) { + return new LazyIdMapBuilder(readConcurrency, true, true); } void update( @@ -454,8 +369,7 @@ void update( @Nullable MapValue targetNodePropertyValues, NodeLabelToken sourceNodeLabels, NodeLabelToken targetNodeLabels, - AnyValue relationshipConfig, - ConfigValidator configValidator + AnyValue relationshipConfig ) { MapValue relationshipProperties = null; RelationshipType relationshipType = RelationshipType.ALL_RELATIONSHIPS; @@ -463,30 +377,39 @@ void update( if (relationshipConfig instanceof MapValue) { relationshipProperties = propertiesConfig("properties", (MapValue) relationshipConfig); relationshipType = typeConfig("relationshipType", (MapValue) relationshipConfig); - - configValidator.validateRelationshipsConfig((MapValue) relationshipConfig); } var intermediateSourceId = loadNode(sourceNode, sourceNodeLabels, sourceNodePropertyValues); if (targetNode != NoValue.NO_VALUE) { - var relImporter = this.relImporters.computeIfAbsent(relationshipType, this::newRelImporter); + RelationshipsBuilder relImporter; + // we do the check before to avoid having to create a new lambda instance on every call + if (this.relImporters.containsKey(relationshipType)) { + relImporter = this.relImporters.get(relationshipType); + } else { + var finalRelationshipProperties = relationshipProperties; + relImporter = this.relImporters.computeIfAbsent( + relationshipType, + type -> newRelImporter(type, finalRelationshipProperties) + ); + } + var intermediateTargetId = loadNode(targetNode, targetNodeLabels, targetNodePropertyValues); - if (this.relationshipPropertySchemas != null) { - assert relationshipProperties != null; - if (this.relationshipPropertySchemas.size() == 1) { - var relationshipProperty = this.relationshipPropertySchemas.get(0).key(); - double propertyValue = loadOneRelationshipProperty( - relationshipProperties, - relationshipProperty - ); - relImporter.addFromInternal(intermediateSourceId, intermediateTargetId, propertyValue); + if (relationshipProperties != null) { + if (relationshipProperties.size() == 1) { + relationshipProperties.foreach((key, value) -> { + var property = ReadHelper.extractValue(value, DefaultValue.DOUBLE_DEFAULT_FALLBACK); + relImporter.addFromInternal(intermediateSourceId, intermediateTargetId, property); + }); } else { - var propertyValues = loadMultipleRelationshipProperties( - relationshipProperties, - this.relationshipPropertySchemas - ); + var propertyValues = new double[relationshipProperties.size()]; + int[] index = {0}; + relationshipProperties.foreach((key, value) -> { + var property = ReadHelper.extractValue(value, DefaultValue.DOUBLE_DEFAULT_FALLBACK); + var i = index[0]++; + propertyValues[i] = property; + }); relImporter.addFromInternal(intermediateSourceId, intermediateTargetId, propertyValues); } } else { @@ -529,8 +452,7 @@ AggregationResult result(String username, DatabaseId databaseId, ProgressTimer t .build(); } - private RelationshipsBuilder newRelImporter(RelationshipType relType) { - + private RelationshipsBuilder newRelImporter(RelationshipType relType, @Nullable MapValue properties) { var undirectedTypes = this.config.undirectedRelationshipTypes(); var orientation = undirectedTypes.contains(relType.name) || undirectedTypes.contains("*") ? UNDIRECTED @@ -542,32 +464,18 @@ private RelationshipsBuilder newRelImporter(RelationshipType relType) { var relationshipsBuilderBuilder = GraphFactory.initRelationshipsBuilder() .nodes(this.idMapBuilder) + .relationshipType(relType) .orientation(orientation) .aggregation(Aggregation.NONE) .indexInverse(indexInverse) .concurrency(this.config.readConcurrency()); - // There is a potential race between initializing the relationships builder and the - // relationship property schemas. Both happen under lock, but under different ones. - // Relationship builders are initialized as part of computeIfAbsent which uses the - // lock inside ConcurrentHashMap, while `this.relationshipPropertySchemas` is initialized - // using the lock in this class. - // - // We have to ensure that the property schemas field is fully initialized, before we - // create the relationships builder. This can only be achieved by using the same lock - // for both actions. This should not affect performance, as we are doing this inside of - // computeIfAbsent which is only called once. - this.lock.lock(); - try { - if (this.relationshipPropertySchemas != null) { - for (var relationshipPropertySchema : this.relationshipPropertySchemas) { - relationshipsBuilderBuilder.addPropertyConfig( - GraphFactory.PropertyConfig.of(relationshipPropertySchema.key()) - ); - } + if (properties != null) { + for (String propertyKey : properties.keySet()) { + relationshipsBuilderBuilder.addPropertyConfig( + ImmutablePropertyConfig.builder().propertyKey(propertyKey).build() + ); } - } finally { - this.lock.unlock(); } return relationshipsBuilderBuilder.build(); @@ -595,91 +503,27 @@ private long loadNode( ); } - private static double loadOneRelationshipProperty( - @NotNull MapValue relationshipProperties, - String relationshipPropertyKey - ) { - var propertyValue = relationshipProperties.get(relationshipPropertyKey); - return ReadHelper.extractValue(propertyValue, DefaultValue.DOUBLE_DEFAULT_FALLBACK); - } - - private static double[] loadMultipleRelationshipProperties( - @NotNull MapValue relationshipProperties, - List relationshipPropertyKeys - ) { - var propertyValues = new double[relationshipPropertyKeys.size()]; - Arrays.setAll(propertyValues, i -> { - var relationshipPropertyKey = relationshipPropertyKeys.get(i).key(); - return loadOneRelationshipProperty(relationshipProperties, relationshipPropertyKey); - }); - return propertyValues; - } - private AdjacencyCompressor.ValueMapper buildNodesWithProperties(GraphStoreBuilder graphStoreBuilder) { - var idMapAndProperties = this.idMapBuilder.build(); - var nodes = idMapAndProperties.idMap(); - - var maybeNodeProperties = idMapAndProperties.nodeProperties(); - - var nodesBuilder = ImmutableNodes.builder().idMap(nodes); - - var nodePropertySchema = maybeNodeProperties - .map(nodeProperties -> nodeSchemaWithProperties( - nodes.availableNodeLabels(), - nodeProperties - )) - .orElseGet(() -> nodeSchemaWithoutProperties(nodes.availableNodeLabels())) - .unionProperties(); - - NodeSchema nodeSchema = NodeSchema.empty(); - nodes.availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); - nodePropertySchema.forEach((propertyKey, propertySchema) -> { - nodes.availableNodeLabels().forEach(label -> { - nodeSchema.getOrCreateLabel(label).addProperty(propertySchema.key(), propertySchema); - }); - }); + + var idMap = idMapAndProperties.idMap(); + var nodeSchema = idMapAndProperties.schema(); this.graphSchemaBuilder.nodeSchema(nodeSchema); - maybeNodeProperties.ifPresent(allNodeProperties -> { - CSRGraphStoreUtil.extractNodeProperties( - nodesBuilder, - nodePropertySchema::get, - allNodeProperties - ); - }); + var nodes = ImmutableNodes + .builder() + .idMap(idMap) + .schema(nodeSchema) + .properties(idMapAndProperties.propertyStore()) + .build(); - graphStoreBuilder.nodes(nodesBuilder.build()); + graphStoreBuilder.nodes(nodes); // Relationships are added using their intermediate node ids. // In order to map to the final internal ids, we need to use // the mapping function of the wrapped id map. - return nodes.rootIdMap()::toMappedNodeId; - } - - private static NodeSchema nodeSchemaWithProperties( - Iterable nodeLabels, - Map propertyMap - ) { - var nodeSchema = NodeSchema.empty(); - - nodeLabels.forEach((nodeLabel) -> { - propertyMap.forEach((propertyName, nodeProperties) -> { - nodeSchema.getOrCreateLabel(nodeLabel).addProperty( - propertyName, - nodeProperties.valueType() - ); - }); - }); - - return nodeSchema; - } - - private static NodeSchema nodeSchemaWithoutProperties(Iterable nodeLabels) { - var nodeSchema = NodeSchema.empty(); - nodeLabels.forEach(nodeSchema::getOrCreateLabel); - return nodeSchema; + return idMap.rootIdMap()::toMappedNodeId; } private void buildRelationshipsWithProperties( @@ -687,21 +531,17 @@ private void buildRelationshipsWithProperties( AdjacencyCompressor.ValueMapper valueMapper ) { var relationshipImportResultBuilder = RelationshipImportResult.builder(); - var relationshipSchemas = new ArrayList(); + var relationshipSchema = MutableRelationshipSchema.empty(); this.relImporters.forEach((relationshipType, relImporter) -> { var relationships = relImporter.build( Optional.of(valueMapper), Optional.empty() ); - relationshipSchemas.add(relationships.relationshipSchema(relationshipType)); + relationshipSchema.set(relationships.relationshipSchemaEntry()); relationshipImportResultBuilder.putImportResult(relationshipType, relationships); }); - var relationshipSchema = relationshipSchemas - .stream() - .reduce(RelationshipSchema.empty(), RelationshipSchema::union); - graphStoreBuilder.relationshipImportResult(relationshipImportResultBuilder.build()); this.graphSchemaBuilder.relationshipSchema(relationshipSchema); @@ -716,9 +556,15 @@ static MapValue propertiesConfig( @NotNull MapValue propertiesConfig ) { var nodeProperties = propertiesConfig.get(propertyKey); + if (nodeProperties instanceof MapValue) { - return (MapValue) nodeProperties; + var mapProperties = (MapValue) nodeProperties; + if (mapProperties.isEmpty()) { + return null; + } + return mapProperties; } + if (nodeProperties == NoValue.NO_VALUE) { return null; } diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index d10edf1cfd8..3f997bb4afc 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -162,6 +162,50 @@ void testArbitraryIds() { } } + @Test + void testDifferentPropertySchemas() { + var query = "UNWIND [" + + " [0, 1, 'a', {}, 'rel', {}], " + + " [2, 3, 'b', {x:1}, 'rel2', {weight: 0.1}]," + + " [5, 6, 'c', {y:1}, 'rel3', {hq: 0.1}]" + + "] AS data" + + " RETURN gds.alpha.graph.project(" + + " 'g'," + + " data[0]," + + " data[1]," + + " {sourceNodeLabels: data[2], sourceNodeProperties: data[3]}," + + " {relationshipType: data[4], properties: data[5]}," + + " {}" + + ")"; + + runQuery(query); + + var graph = GraphStoreCatalog.get("", db.databaseName(), "g").graphStore().getUnion(); + + assertThat(graph.schema().nodeSchema().get(NodeLabel.of("a")).properties().keySet()).isEmpty(); + assertThat(graph.schema().nodeSchema().get(NodeLabel.of("b")).properties().keySet()).containsExactly("x"); + assertThat(graph.schema().nodeSchema().get(NodeLabel.of("c")).properties().keySet()).containsExactly("y"); + + assertThat(graph + .schema() + .relationshipSchema() + .get(org.neo4j.gds.RelationshipType.of("rel")) + .properties() + .keySet()).isEmpty(); + assertThat(graph + .schema() + .relationshipSchema() + .get(org.neo4j.gds.RelationshipType.of("rel2")) + .properties() + .keySet()).containsExactly("weight"); + assertThat(graph + .schema() + .relationshipSchema() + .get(org.neo4j.gds.RelationshipType.of("rel3")) + .properties() + .keySet()).containsExactly("hq"); + } + @ParameterizedTest @CsvSource({"13.37, Double", "true, Boolean", "false, Boolean", "null, NO_VALUE", "\"42\", String", "[42], List", "[13.37], List", "{foo:42}, Map", "{foo:13.37}, Map"}) void testInvalidArbitraryIds(String idLiteral, String invalidType) { @@ -174,6 +218,13 @@ void testInvalidArbitraryIds(String idLiteral, String invalidType) { .hasMessage("The node has to be either a NODE or an INTEGER, but got " + invalidType); } + @Test + void testInvalidNegativId() { + assertThatThrownBy(() -> runQuery("RETURN gds.alpha.graph.project('g', -1)")) + .rootCause() + .hasMessage("GDS expects node ids to be positive. But got a negative id of `-1`."); + } + @Test void testInvalidRelationshipAsArbitraryId() { var query = "MATCH ()-[r]-() RETURN gds.alpha.graph.project('g', r)"; @@ -640,7 +691,7 @@ void testRespectUndirectedTypes(List undirectedConfig, List expe .relationshipSchema() .entries() .forEach(entry -> { - var expectedDirection = expectedUndirectedTypes.contains(entry.identifier.name) ? org.neo4j.gds.api.schema.Direction.UNDIRECTED : org.neo4j.gds.api.schema.Direction.DIRECTED; + var expectedDirection = expectedUndirectedTypes.contains(entry.identifier().name) ? org.neo4j.gds.api.schema.Direction.UNDIRECTED : org.neo4j.gds.api.schema.Direction.DIRECTED; assertThat(entry.direction()).isEqualTo(expectedDirection); }); } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/CypherGraphStore.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/CypherGraphStore.java index 3cfd28b3210..571b9e828e9 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/CypherGraphStore.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/CypherGraphStore.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.cypher; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.GraphStoreAdapter; import org.neo4j.gds.api.IdMap; @@ -102,18 +101,17 @@ public void addNodeProperty( @Override - public void addRelationshipType( - RelationshipType relationshipType, - SingleTypeRelationships relationships - ) { - innerGraphStore().addRelationshipType(relationshipType, relationships); + public void addRelationshipType(SingleTypeRelationships relationships) { + innerGraphStore().addRelationshipType(relationships); relationships.properties().ifPresent( properties -> properties .keySet() .forEach(propertyKey -> stateVisitors.forEach(stateVisitor -> stateVisitor.relationshipPropertyAdded( propertyKey))) ); - stateVisitors.forEach(stateVisitor -> stateVisitor.relationshipTypeAdded(relationshipType.name())); + + var relType = relationships.relationshipSchemaEntry().identifier(); + stateVisitors.forEach(stateVisitor -> stateVisitor.relationshipTypeAdded(relType.name)); } public RelationshipIds relationshipIds() { diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java index 456a0c2ff70..436bab6510e 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java @@ -36,7 +36,7 @@ public UpdatableDoubleArrayNodeProperty(long nodeCount, double[] defaultValue) { } @Override - public long size() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java index 2b486482a53..5534a4f8438 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java @@ -35,9 +35,8 @@ public UpdatableDoubleNodeProperty(long nodeCount, double defaultValue) { this.doubleList = HugeSparseDoubleList.of(defaultValue); } - @Override - public long size() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java index 1ccaa4dd465..6075249a460 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java @@ -36,7 +36,7 @@ public UpdatableFloatArrayNodeProperty(long nodeCount, float[] defaultValue) { } @Override - public long size() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java index 63b7475c967..7ae269f5a64 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java @@ -36,7 +36,7 @@ public UpdatableLongArrayNodeProperty(long nodeCount, long[] defaultValue) { } @Override - public long size() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java index 7612386d2f6..b64868d265e 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java @@ -35,9 +35,8 @@ public UpdatableLongNodeProperty(long nodeCount, long defaultValue) { this.longList = HugeSparseLongList.of(defaultValue); } - @Override - public long size() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-test/src/test/java/org/neo4j/gds/storageengine/InMemoryNodeCursorTest.java b/cypher/cypher-test/src/test/java/org/neo4j/gds/storageengine/InMemoryNodeCursorTest.java index d00f8d603ca..70ea3a26862 100644 --- a/cypher/cypher-test/src/test/java/org/neo4j/gds/storageengine/InMemoryNodeCursorTest.java +++ b/cypher/cypher-test/src/test/java/org/neo4j/gds/storageengine/InMemoryNodeCursorTest.java @@ -22,7 +22,6 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; @@ -53,8 +52,8 @@ protected void onSetup() { protected GraphStore graphStore() { return new StoreLoaderBuilder() .databaseService(db) - .addNodeProjection(NodeProjection.of("A", PropertyMappings.of(PropertyMapping.of("prop1")))) - .addNodeProjection(NodeProjection.of("B", PropertyMappings.of(PropertyMapping.of("prop2")))) + .addNodeProjection(NodeProjection.builder().label("A").addProperty(PropertyMapping.of("prop1")).build()) + .addNodeProjection(NodeProjection.builder().label("B").addProperty(PropertyMapping.of("prop2")).build()) .build() .graphStore(); } diff --git a/cypher/cypher-test/src/test/java/org/neo4j/internal/recordstorage/InMemoryStorageEngineTest.java b/cypher/cypher-test/src/test/java/org/neo4j/internal/recordstorage/InMemoryStorageEngineTest.java index 336f9a77309..dba53ca9cf3 100644 --- a/cypher/cypher-test/src/test/java/org/neo4j/internal/recordstorage/InMemoryStorageEngineTest.java +++ b/cypher/cypher-test/src/test/java/org/neo4j/internal/recordstorage/InMemoryStorageEngineTest.java @@ -23,7 +23,6 @@ import org.neo4j.gds.NodeProjection; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.GraphStore; @@ -45,8 +44,8 @@ class InMemoryStorageEngineTest extends CypherTest { protected GraphStore graphStore() { return new StoreLoaderBuilder() .databaseService(db) - .addNodeProjection(NodeProjection.of("A", PropertyMappings.of(PropertyMapping.of("prop1")))) - .addNodeProjection(NodeProjection.of("B", PropertyMappings.of(PropertyMapping.of("prop2")))) + .addNodeProjection(NodeProjection.builder().label("A").addProperty(PropertyMapping.of("prop1")).build()) + .addNodeProjection(NodeProjection.builder().label("B").addProperty(PropertyMapping.of("prop2")).build()) .addRelationshipProjection(RelationshipProjection.of("REL", Orientation.NATURAL)) .build() .graphStore(); diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoChecker.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoChecker.java index baab4ddbd7c..50218ee7fc0 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoChecker.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoChecker.java @@ -166,6 +166,8 @@ private Consumer assertTableValues( .getCells() .get(0)) // Get the first column in the row --> corresponds to the return column names .map(Cell::getText) + // remove any potential links in the names + .map(name -> name.replaceAll("|<\\/a>", "")) // as java identifier cannot contain white spaces, remove anything after the first space such as footnote: .map(name -> name.split("\\s+")[0]) .collect(Collectors.toList()); @@ -176,7 +178,7 @@ private Consumer assertTableValues( }; } - private Iterable extractDocResultFields(String syntaxCode) { + private static Iterable extractDocResultFields(String syntaxCode) { var yield = syntaxCode.substring(syntaxCode.indexOf(YIELD_KEYWORD) + YIELD_KEYWORD.length()).trim(); return Arrays.stream(yield.split(YIELD_FIELD_SEPARATOR)) .map(yieldField -> yieldField.split(YIELD_NAME_DATA_TYPE_SEPARATOR)[0].trim()) diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java index a200a2ed2dd..8e80e69244d 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java @@ -66,8 +66,9 @@ public enum SyntaxMode { SYSTEM_MONITOR("system-monitor-syntax", false), SYS_INFO("debug-sysinfo-syntax", false), WRITE_NODE_LABEL("include-with-write-node-label", false), - MUTATE_NODE_LABEL("include-with-mutate-node-label", false),; - + MUTATE_NODE_LABEL("include-with-mutate-node-label", false), + GRAPH_GENERATE("include-with-graph-generate"),; + private final String mode; public final boolean hasParameters; diff --git a/doc-test-tools/src/test/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoCheckerTest.java b/doc-test-tools/src/test/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoCheckerTest.java index 9e439cce525..5d9838e0e65 100644 --- a/doc-test-tools/src/test/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoCheckerTest.java +++ b/doc-test-tools/src/test/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoCheckerTest.java @@ -23,11 +23,14 @@ import org.asciidoctor.OptionsBuilder; import org.asciidoctor.SafeMode; import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.File; import java.net.URISyntaxException; @@ -48,6 +51,9 @@ class ProcedureSyntaxAutoCheckerTest { private static final String newLine = System.lineSeparator(); + @InjectSoftAssertions + private SoftAssertions softAssertions; + @BeforeEach void setUp() { // By default we are forced to use relative path which we don't want. @@ -57,10 +63,11 @@ void setUp() { } - @Test - void correctSyntaxSectionTest(SoftAssertions softAssertions) throws URISyntaxException { + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"include-with-syntax.adoc", "include-with-syntax-parameter-with-link.adoc"}) + void correctSyntaxSectionTest(String positiveResource) throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { - var file = Paths.get(getClass().getClassLoader().getResource("include-with-syntax.adoc").toURI()).toFile(); + var file = Paths.get(getClass().getClassLoader().getResource(positiveResource).toURI()).toFile(); assertTrue(file.exists() && file.canRead()); asciidoctor.convertFile(file, options); @@ -68,7 +75,7 @@ void correctSyntaxSectionTest(SoftAssertions softAssertions) throws URISyntaxExc } @Test - void shouldFailOnMissingResultsTable(SoftAssertions softAssertions) throws URISyntaxException { + void shouldFailOnMissingResultsTable() throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { var file = Paths .get(getClass() @@ -85,7 +92,7 @@ void shouldFailOnMissingResultsTable(SoftAssertions softAssertions) throws URISy } @Test - void shouldFailOnMoreThanOneResultsTable(SoftAssertions softAssertions) throws URISyntaxException { + void shouldFailOnMoreThanOneResultsTable() throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { var file = Paths .get(getClass() @@ -102,7 +109,7 @@ void shouldFailOnMoreThanOneResultsTable(SoftAssertions softAssertions) throws U } @Test - void shouldFailOnMissingCodeBlock(SoftAssertions softAssertions) throws URISyntaxException { + void shouldFailOnMissingCodeBlock() throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { var file = Paths .get(getClass().getClassLoader().getResource("invalid-include-with-syntax-no-code-block.adoc").toURI()) @@ -116,7 +123,7 @@ void shouldFailOnMissingCodeBlock(SoftAssertions softAssertions) throws URISynta } @Test - void shouldFailOnMoreThanOneCodeBlock(SoftAssertions softAssertions) throws URISyntaxException { + void shouldFailOnMoreThanOneCodeBlock() throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { var file = Paths .get(getClass() diff --git a/doc-test-tools/src/test/resources/include-with-syntax-parameter-with-link.adoc b/doc-test-tools/src/test/resources/include-with-syntax-parameter-with-link.adoc new file mode 100644 index 00000000000..4d1f95c7d18 --- /dev/null +++ b/doc-test-tools/src/test/resources/include-with-syntax-parameter-with-link.adoc @@ -0,0 +1,42 @@ +[.include-with-stream] +====== +.Run Louvain in stream mode on a named graph. +[source, cypher, role=noplay] +---- +CALL gds.louvain.stream( + graphName: String, + configuration: Map +) +YIELD + nodeId: Integer, + communityId: Integer, + intermediateCommunityIds: List of Integer +---- + +.Parameters +[opts="header"] +|=== +| Name +| graphName footnote:vital[choose the name wisely] +| <> +|=== + +// This table is only here to make sure we will really pick the `.Results` one +[[configuration-table]] +.Algorithm specific configuration +[opts="header",cols="1,1,1m,1,4"] +|=== +| Name | Type | Default | Optional | Description +| relationshipWeightProperty | String | null | yes | Relationship Weight. +| seedProperty | String | n/a | yes | Seed Property. +|=== + +.Results +[opts="header",cols="1,1,6"] +|=== +| Name | Type | Description +| nodeId | Integer | Node ID. +| communityId | Integer | The community ID of the final level. +| intermediateCommunityIds | List of Integer | Community IDs for each level. `Null` if `includeIntermediateCommunities` is set to false. +|=== +====== diff --git a/doc-test/build.gradle b/doc-test/build.gradle index 9f15dc4dbb7..20f18c81bcf 100644 --- a/doc-test/build.gradle +++ b/doc-test/build.gradle @@ -1,6 +1,6 @@ apply plugin: 'java-library' -description = 'Neo4j Graph Data Science :: Docs' +description = 'Neo4j Graph Data Science :: Documentation Testing' group = 'org.neo4j.gds' @@ -41,3 +41,7 @@ tasks.register('unpackDocs', Copy) { } processTestResources.dependsOn tasks.named('unpackDocs') + +tasks.test { + enabled = !rootProject.hasProperty('aurads') +} diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/AllPairsShortestPathDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/AllPairsShortestPathDocTest.java new file mode 100644 index 00000000000..21dbcb12a0d --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/AllPairsShortestPathDocTest.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc; + +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.functions.AsNodeFunc; +import org.neo4j.gds.functions.IsFiniteFunc; +import org.neo4j.gds.projection.CypherAggregation; +import org.neo4j.gds.shortestpaths.AllShortestPathsProc; + +import java.util.List; + +class AllPairsShortestPathDocTest extends SingleFileDocTestBase { + + @Override + protected List> procedures() { + return List.of( + GraphProjectProc.class, + CypherAggregation.class, + AllShortestPathsProc.class + ); + } + + @Override + protected List> functions() { + return List.of(AsNodeFunc.class, IsFiniteFunc.class); + } + + @Override + protected String adocFile() { + return "pages/alpha-algorithms/all-pairs-shortest-path.adoc"; + } +} diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/CELFDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/CELFDocTest.java index ad0b1620170..5ac7bd6cbd5 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/CELFDocTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/CELFDocTest.java @@ -50,6 +50,6 @@ protected List> procedures() { @Override protected String adocFile() { - return "pages/algorithms/influence-maximization/celf.adoc"; + return "pages/algorithms/celf.adoc"; } } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java new file mode 100644 index 00000000000..18b3212ebc4 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc; + +import org.neo4j.gds.beta.generator.GraphGenerateProc; +import org.neo4j.gds.catalog.GraphStreamRelationshipPropertiesProc; +import org.neo4j.gds.catalog.GraphStreamRelationshipsProc; +import org.neo4j.gds.functions.AsNodeFunc; + +import java.util.List; + +class GraphGenerationDocTest extends SingleFileDocTestBase { + + @Override + protected List> functions() { + return List.of(AsNodeFunc.class); + } + + @Override + protected List> procedures() { + return List.of(GraphGenerateProc.class, GraphStreamRelationshipsProc.class, + GraphStreamRelationshipPropertiesProc.class + ); + } + + @Override + protected String adocFile() { + return "pages/management-ops/projections/graph-generation.adoc"; + } +} diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/KSpanningTreeDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/KSpanningTreeDocTest.java new file mode 100644 index 00000000000..4d11e82ae66 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/KSpanningTreeDocTest.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc; + +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.functions.AsNodeFunc; +import org.neo4j.gds.spanningtree.KSpanningWriteTreeProc; + +import java.util.List; + +class KSpanningTreeDocTest extends SingleFileDocTestBase { + + @Override + protected List> functions() { + return List.of(AsNodeFunc.class); + } + + @Override + protected List> procedures() { + return List.of( + KSpanningWriteTreeProc.class, + GraphProjectProc.class + ); + } + + @Override + protected String adocFile() { + return "pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc"; + } + +} diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java index 11cc2253e20..e1e097effdf 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java @@ -39,6 +39,6 @@ protected Iterable syntaxModes() { @Override protected String adocFile() { - return "pages/algorithms/influence-maximization/celf.adoc"; + return "pages/algorithms/celf.adoc"; } } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/SingleElementIterator.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java similarity index 61% rename from cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/SingleElementIterator.java rename to doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java index 8e871d05b80..20ca0ed6b04 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/SingleElementIterator.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java @@ -17,26 +17,21 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.gds.core.cypher; +package org.neo4j.gds.doc.syntax; -import com.carrotsearch.hppc.AbstractIterator; +import java.util.List; -public class SingleElementIterator extends AbstractIterator { +import static org.neo4j.gds.doc.syntax.SyntaxMode.GRAPH_GENERATE; - private final T element; - private boolean firstIteration; +class GraphGenerationSyntaxTest extends SyntaxTestBase { - public SingleElementIterator(T element) { - this.element = element; - this.firstIteration = true; + @Override + protected Iterable syntaxModes() { + return List.of(SyntaxModeMeta.of(GRAPH_GENERATE)); } @Override - protected T fetch() { - if (!firstIteration) { - return done(); - } - firstIteration = false; - return element; + protected String adocFile() { + return "pages/management-ops/projections/graph-generation.adoc"; } } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/KSpanningTreeSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/KSpanningTreeSyntaxTest.java new file mode 100644 index 00000000000..af085dd7540 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/KSpanningTreeSyntaxTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc.syntax; + +import java.util.List; + +import static org.neo4j.gds.doc.syntax.SyntaxMode.WRITE; + +class KSpanningTreeSyntaxTest extends SyntaxTestBase { + + protected Iterable syntaxModes() { + return List.of( + SyntaxModeMeta.of(WRITE) + ); + } + + @Override + protected String adocFile() { + return "pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc"; + } +} diff --git a/doc/antora.yml b/doc/antora.yml index dad9b547855..72675a5fc1b 100644 --- a/doc/antora.yml +++ b/doc/antora.yml @@ -1,6 +1,6 @@ name: graph-data-science title: Neo4j Graph Data Science -version: '2.3-preview' +version: '2.3' start_page: ROOT:index.adoc nav: - modules/ROOT/content-nav.adoc diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index cc705b62235..71191a6c002 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -6,7 +6,7 @@ ** xref:installation/neo4j-server.adoc[] ** xref:installation/installation-enterprise-edition.adoc[] ** xref:installation/installation-docker.adoc[] -** xref:installation/installation-causal-cluster.adoc[] +** xref:installation/installation-neo4j-cluster.adoc[] ** xref:installation/installation-apache-arrow.adoc[] ** xref:installation/additional-config-parameters.adoc[] ** xref:installation/System-requirements.adoc[] @@ -38,7 +38,6 @@ ** xref:management-ops/create-cypher-db.adoc[] ** xref:management-ops/administration.adoc[] ** xref:management-ops/backup-restore.adoc[] -** xref:management-ops/defaults-and-limits.adoc[] * xref:algorithms/index.adoc[] ** xref:algorithms/syntax.adoc[] ** xref:algorithms/centrality.adoc[] @@ -50,8 +49,7 @@ *** xref:algorithms/closeness-centrality.adoc[] *** xref:algorithms/harmonic-centrality.adoc[] *** xref:algorithms/hits.adoc[] -*** xref:algorithms/influence-maximization.adoc[] -**** xref:algorithms/influence-maximization/celf.adoc[] +*** xref:algorithms/celf.adoc[] ** xref:algorithms/community.adoc[] *** xref:algorithms/louvain.adoc[] *** xref:algorithms/label-propagation.adoc[] @@ -145,17 +143,21 @@ * xref:end-to-end-examples/end-to-end-examples.adoc[] ** xref:end-to-end-examples/fastrp-knn-example.adoc[] * xref:production-deployment/index.adoc[] +** xref:production-deployment/defaults-and-limits.adoc[] ** xref:production-deployment/transaction-handling.adoc[] -** xref:production-deployment/fabric.adoc[] -** xref:production-deployment/causal-cluster.adoc[] +** xref:production-deployment/composite.adoc[] +** xref:production-deployment/neo4j-cluster.adoc[] +** xref:production-deployment/configuration-settings.adoc[] ** xref:production-deployment/feature-toggles.adoc[] * xref:python-client/index.adoc[] +* link:https://neo4j.com/docs/bloom-user-guide/current/bloom-tutorial/gds-integration/[Bloom visualization] * Appendix ** xref:operations-reference/appendix-a.adoc[] *** xref:operations-reference/graph-operation-references.adoc[] *** xref:operations-reference/algorithm-references.adoc[] *** xref:operations-reference/machine-learning-references.adoc[] *** xref:operations-reference/additional-operation-references.adoc[] +*** xref:operations-reference/configuration-settings.adoc[] ** xref:appendix-b/index.adoc[] *** xref:appendix-b/migration-algos-common.adoc[] *** xref:appendix-b/migration-graph-projection.adoc[] diff --git a/doc/modules/ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc b/doc/modules/ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc index 80f1edebfc3..27c53ac4838 100644 --- a/doc/modules/ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc +++ b/doc/modules/ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc @@ -342,7 +342,7 @@ YIELD nodeCount, relationshipCount, bytesMin, bytesMax, requiredMemory [opts="header",cols="1,1,1,1,1"] |=== | nodeCount | relationshipCount | bytesMin | bytesMax | requiredMemory -| 9 | 9 | 2528 | 2744 | "[2528 Bytes \... 2744 Bytes]" +| 9 | 9 | 2384 | 2600 | "[2384 Bytes \... 2600 Bytes]" |=== -- [[algorithms-filtered-node-similarity-examples-stream]] diff --git a/doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc b/doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc index 3734c8b0254..33b2e2ce995 100644 --- a/doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc +++ b/doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc @@ -185,6 +185,8 @@ For more details on the stream mode in general, see xref:common-usage/running-al ---- CALL gds.alpha.modularity.stream('myGraph', { communityProperty: 'community', relationshipWeightProperty: 'weight' }) YIELD communityId, modularity +RETURN communityId, modularity +ORDER BY communityId ASC ---- .Results diff --git a/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc b/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc index d89d2a2dd56..8702d3d3859 100644 --- a/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc +++ b/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc @@ -30,7 +30,7 @@ The implementation requires _O(n + m)_ space and runs in _O(n * m)_ time, where For more information on this algorithm, see: -* https://www.eecs.wsu.edu/~assefaw/CptS580-06/papers/brandes01centrality.pdf[A Faster Algorithm for Betweenness Centrality^] +* https://snap.stanford.edu/class/cs224w-readings/brandes01centrality.pdf[A Faster Algorithm for Betweenness Centrality^] * https://www.uni-konstanz.de/mmsp/pubsys/publishedFiles/BrPi07.pdf[Centrality Estimation in Large Networks^] * http://moreno.ss.uci.edu/23.pdf[A Set of Measures of Centrality Based on Betweenness^] diff --git a/doc/modules/ROOT/pages/algorithms/influence-maximization/celf.adoc b/doc/modules/ROOT/pages/algorithms/celf.adoc similarity index 99% rename from doc/modules/ROOT/pages/algorithms/influence-maximization/celf.adoc rename to doc/modules/ROOT/pages/algorithms/celf.adoc index d3aa3b1e0d1..f9b0cdac1de 100644 --- a/doc/modules/ROOT/pages/algorithms/influence-maximization/celf.adoc +++ b/doc/modules/ROOT/pages/algorithms/celf.adoc @@ -1,3 +1,6 @@ +:page-aliases: algorithms/influence-maximization/celf.adoc +:page-aliases: algorithms/influence-maximization/ + [[algorithms-celf]] [.beta] = CELF diff --git a/doc/modules/ROOT/pages/algorithms/centrality.adoc b/doc/modules/ROOT/pages/algorithms/centrality.adoc index b9d6f5541b7..0d04f0b6359 100644 --- a/doc/modules/ROOT/pages/algorithms/centrality.adoc +++ b/doc/modules/ROOT/pages/algorithms/centrality.adoc @@ -14,7 +14,8 @@ The Neo4j GDS library includes the following centrality algorithms, grouped by q ** xref:algorithms/degree-centrality.adoc[Degree Centrality] * Beta ** xref:algorithms/closeness-centrality.adoc[Closeness Centrality] +** xref:algorithms/celf.adoc[CELF] * Alpha ** xref:algorithms/harmonic-centrality.adoc[Harmonic Centrality] ** xref:algorithms/hits.adoc[HITS] -** xref:algorithms/influence-maximization.adoc[Influence Maximization] + diff --git a/doc/modules/ROOT/pages/algorithms/dfs.adoc b/doc/modules/ROOT/pages/algorithms/dfs.adoc index f24525602ce..5e288f693ac 100644 --- a/doc/modules/ROOT/pages/algorithms/dfs.adoc +++ b/doc/modules/ROOT/pages/algorithms/dfs.adoc @@ -4,7 +4,7 @@ :entity: relationship :result: path in traversal order :algorithm: Depth First Search - +:sequential: true :directed: :undirected: diff --git a/doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc b/doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc index b60c6a3f05f..1dbd5ee0bfe 100644 --- a/doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc +++ b/doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc @@ -6,6 +6,7 @@ :algorithm: Dijkstra :source-target: false :procedure-name: pass:q[gds.allShortestPaths.dijkstra] +:sequential: true :directed: @@ -25,7 +26,7 @@ To compute the shortest path between a source and a target node, xref:algorithms The GDS implementation is based on the http://www-m3.ma.tum.de/twiki/pub/MN0506/WebHome/dijkstra.pdf[original description] and uses a binary heap as priority queue. The implementation is also used for the xref:algorithms/astar.adoc[A*] and xref:algorithms/yens.adoc[Yen's] algorithms, as well as xref:algorithms/betweenness-centrality.adoc[weighted Betweenness Centrality]. -The algorithm implementation is executed using a single thread and altering the concurrency configuration has no effect. +The algorithm implementation is executed using a single thread. You can consider xref:algorithms/delta-single-source.adoc[Delta-Stepping] for an efficient parallel shortest path algorithm instead. [[algorithms-dijkstra-single-source-syntax]] diff --git a/doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc b/doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc index c9be2d4323b..8e5386debae 100644 --- a/doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc +++ b/doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc @@ -6,6 +6,7 @@ :algorithm: Dijkstra :source-target: true :procedure-name: pass:q[gds.shortestPath.dijkstra] +:sequential: true :directed: diff --git a/doc/modules/ROOT/pages/algorithms/influence-maximization.adoc b/doc/modules/ROOT/pages/algorithms/influence-maximization.adoc deleted file mode 100644 index 798127310d6..00000000000 --- a/doc/modules/ROOT/pages/algorithms/influence-maximization.adoc +++ /dev/null @@ -1,11 +0,0 @@ -[[algorithms-influence-maximization]] -= Influence Maximization -:description: This chapter provides explanations and examples for each of the influence maximization algorithms in the Neo4j Graph Data Science library. - - -The objective of influence maximization is to find a small subset of k nodes from a network in order to achieve maximization to the total number of nodes influenced by these k nodes. -The Neo4j GDS library includes the following alpha influence maximization algorithms: - -* Beta -** xref:algorithms/influence-maximization/celf.adoc[CELF] - diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 78e4bfe1627..e080aa6a5ef 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -1,3 +1,4 @@ +:page-aliases: algorithms/alpha/kmeans.adoc [[algorithms-k-means]] [.beta] = K-Means Clustering @@ -14,13 +15,24 @@ include::partial$/operations-reference/beta-note.adoc[] K-Means clustering is an unsupervised learning algorithm that is used to solve clustering problems. It follows a simple procedure of classifying a given data set into a number of clusters, defined by the parameter `k`. -The clusters are then positioned as points and all observations or data points are associated with the nearest cluster, computed, adjusted and then the process starts over using the new adjustments until a desired result is reached. +The Neo4j GDS Library conducts clustering based on node properties, with a float array node property being passed as input via the `nodeProperty` parameter. +Nodes in the graph are then positioned as points in a `d`-dimensional space (where `d` is the length of the array property). + +The algorithm then begins by selecting `k` initial cluster centroids, which are `d`-dimensional arrays (see xref:algorithms-kmeans-sampling[section below] for more details). +The centroids act as representatives for a cluster. + +Then, all nodes in the graph calculate their Euclidean distance from each of the cluster centroids and are assigned to the cluster of minimum distance from them. + After these assignments, each cluster takes the mean of all nodes (as points) assigned to it to form its new representative centroid (as a `d`-dimensional array). + +The process repeats with the new centroids until results stabilize, i.e., only a few nodes change clusters per iteration or the number of maximum iterations is reached. + +Note that the K-Means implementation ignores relationships as it is only focused on node properties. For more information on this algorithm, see: * https://en.wikipedia.org/wiki/K-means_clustering -[[algorithms-kmeans-introduction-sampling]] +[[algorithms-kmeans-sampling]] == Initial Centroid Sampling The algorithm starts by picking `k` centroids by randomly sampling from the set of available nodes. diff --git a/doc/modules/ROOT/pages/algorithms/knn.adoc b/doc/modules/ROOT/pages/algorithms/knn.adoc index 67e684d49e6..5f5a6023c3e 100644 --- a/doc/modules/ROOT/pages/algorithms/knn.adoc +++ b/doc/modules/ROOT/pages/algorithms/knn.adoc @@ -7,11 +7,19 @@ :algorithm: K-Nearest Neighbors :knnSpecificConfigurationTableTitle: Algorithm specific configuration - :directed: :undirected: :homogeneous: :weighted: +include::partial$/algorithms/shared/algorithm-traits.adoc[] + +{nbsp} + +[TIP] +==== +kNN is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example] +==== [[algorithms-knn-intro]] == Introduction @@ -19,7 +27,7 @@ The K-Nearest Neighbors algorithm computes a distance value for all node pairs in the graph and creates new relationships between each node and its k nearest neighbors. The distance is calculated based on node properties. -The input of this algorithm is a monopartite graph. +The input of this algorithm is a homogeneous graph. The graph does not need to be connected, in fact, existing relationships between nodes will be ignored - apart from random walk sampling if that that initial sampling option is used. New relationships are created between each node and its k nearest neighbors. diff --git a/doc/modules/ROOT/pages/algorithms/leiden.adoc b/doc/modules/ROOT/pages/algorithms/leiden.adoc index fe642ed62ad..0c903b4a5a8 100644 --- a/doc/modules/ROOT/pages/algorithms/leiden.adoc +++ b/doc/modules/ROOT/pages/algorithms/leiden.adoc @@ -1,4 +1,6 @@ -le[[algorithms-leiden]] +:page-aliases: algorithms/alpha/leiden.adoc + +[[algorithms-leiden]] [.beta] = Leiden :description: This section describes the Leiden algorithm in the Neo4j Graph Data Science library. diff --git a/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc index 06b95851c27..7637d32d767 100644 --- a/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc @@ -1,3 +1,4 @@ +:page-aliases: alpha-algorithms/minimum-weight-spanning-tree.adoc [[algorithms-minimum-weight-spanning-tree]] [.beta] = Minimum Weight Spanning Tree @@ -5,6 +6,7 @@ :entity: relationship :result: weight :algorithm: Prim +:sequential: true include::partial$/operations-reference/beta-note.adoc[] diff --git a/doc/modules/ROOT/pages/algorithms/node-similarity.adoc b/doc/modules/ROOT/pages/algorithms/node-similarity.adoc index 79082b230d6..d03291d9d3d 100644 --- a/doc/modules/ROOT/pages/algorithms/node-similarity.adoc +++ b/doc/modules/ROOT/pages/algorithms/node-similarity.adoc @@ -335,7 +335,7 @@ YIELD nodeCount, relationshipCount, bytesMin, bytesMax, requiredMemory [opts="header",cols="1,1,1,1,1"] |=== | nodeCount | relationshipCount | bytesMin | bytesMax | requiredMemory -| 9 | 9 | 2528 | 2744 | "[2528 Bytes \... 2744 Bytes]" +| 9 | 9 | 2384 | 2600 | "[2384 Bytes \... 2600 Bytes]" |=== -- diff --git a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc index 66451f79d3b..8aedc58a595 100644 --- a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc +++ b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc @@ -29,7 +29,7 @@ The introduction of a new Pregel algorithm can be separated in two main steps. First, we need to implement the algorithm using the Pregel Java API. Second, we need to expose the algorithm via a Cypher procedure to make use of it. -For an example on how to expose a custom Pregel computation via a Neo4j procedure, have a look at the https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-example/src/main/java/org/neo4j/gds/beta/pregel[Pregel examples]. +For an example on how to expose a custom Pregel computation via a Neo4j procedure, have a look at the https://github.com/neo4j/graph-data-science/tree/2.3/examples/pregel-example/src/main/java/org/neo4j/gds/beta/pregel[Pregel examples]. [[algorithms-pregel-api-java]] == Pregel Java API @@ -404,6 +404,45 @@ public class CustomComputation implements PregelComputation { } ---- +[[algorithms-pregel-api-bidirectional]] +=== Traversing incoming relationships + +Some algorithms implemented in Pregel might require or benefit from the ability to access and send messages to all incoming relationships of the current context node. +GDS supports the creation of inverse indexes for relationship types, which enables the traversal of incoming relationships for directed relationship types. + +A Pregel algorithm can access this index by implementing the `org.neo4j.gds.beta.pregel.BidirectionalPregelComputation` interface instead of the `PregelComputation` interface. +Implementing this interface has the following consequences: + +* The Pregel framework will make sure that all relationships passed into the algorithm are inverse indexed. + If no such index exists, an error will be thrown. +* The signature of the `init` and `compute` functions now accept a `org.neo4j.gds.beta.pregel.context.InitContext.BidirectionalInitContext` and `org.neo4j.gds.beta.pregel.context.ComputeContext.BidirectionalComputeContext` respectively. +* Algorithms annotated with the `@PregelProcedure` annotation will automatically create all required inverse indexes. + +The `BidirectionalInitContext` and `BidirectionalComputeContexts` expose the following new methods in addition to the methods defined by `InitContext` and `ComputeContext`: + +[source, java] +---- +//Returns the incoming degree (number of relationships) of the currently processed node. +public int incomingDegree(); +// Calls the consumer for each incoming neighbor of the currently processed node. +public void forEachIncomingNeighbor(LongConsumer targetConsumer); +// Calls the consumer for each incoming neighbor of the given node. +public void forEachIncomingNeighbor(long nodeId, LongConsumer targetConsumer); +// Calls the consumer once for each incoming neighbor of the currently processed node. +public void forEachDistinctIncomingNeighbor(LongConsumer targetConsumer); +// Calls the consumer once for each incoming neighbor of the given node. +public void forEachDistinctIncomingNeighbor(long nodeId, LongConsumer targetConsumer); +---- + +In addition, the `BidirectionalComputeContext` also exposes the following function: + +[source, java] +---- +// Sends the given message to all neighbors of the node. +public void sendToIncomingNeighbors(double message); +---- + + [[algorithms-pregel-api-logging]] === Logging @@ -510,7 +549,7 @@ For more details, please refer to the xref:algorithms/pregel-api.adoc#algorithms === Building and installing a Neo4j plugin In order to use a Pregel algorithm in Neo4j via a procedure, we need to package it as Neo4j plugin. -The https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-bootstrap[pregel-bootstrap] project is a good starting point. +The https://github.com/neo4j/graph-data-science/tree/2.3/examples/pregel-bootstrap[pregel-bootstrap] project is a good starting point. The `build.gradle` file within the project contains all the dependencies necessary to implement a Pregel algorithm and to generate corresponding procedures. Make sure to change the `gdsVersion` and `neo4jVersion` according to your setup. @@ -535,13 +574,11 @@ dbms.security.procedures.unrestricted=custom.pregel.proc.* dbms.security.procedures.allowlist=custom.pregel.proc.* ---- -NOTE: Before `Neo4j 4.2`, the configuration setting is called `dbms.security.procedures.whitelist` - [[algorithms-pregel-api-example]] == Examples -The https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-example[pregel-examples] module contains a set of examples for Pregel algorithms. +The https://github.com/neo4j/graph-data-science/tree/2.3/examples/pregel-example[pregel-examples] module contains a set of examples for Pregel algorithms. The algorithm implementations demonstrate the usage of the Pregel API. Along with each example, we provide test classes that can be used as a guideline on how to write tests for custom algorithms. To play around, we recommend copying one of the algorithms into the `pregel-bootstrap` project, build it and setup the plugin in Neo4j. diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index decc461ec42..0e4c3d3835f 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -190,7 +190,7 @@ include::partial$/algorithms/common-configuration/common-parameters.adoc[] |=== | Name | Type | Default | Optional | Description include::partial$/algorithms/common-configuration/common-write-configuration-entries.adoc[] -include::partial$/algorithms/wcc/specific-configuration.adoc[] +include::partial$/algorithms/wcc/specific-configuration-write.adoc[] |=== .Results @@ -522,7 +522,7 @@ CALL gds.graph.project( _Step 4_ -[role=query-example, group=seeding] +[role=query-example, group=seeding, no-result=true] -- .The following will run the algorithm in `stream` mode using `seedProperty`: [source, cypher, role=noplay] @@ -626,12 +626,12 @@ CALL gds.graph.project( The following query is identical to the stream example in the xref:algorithms/wcc.adoc#algorithms-wcc-examples-seeding[previous section]. This time, we execute WCC on `myIndexedGraph` which will allow the algorithm to use the sampled strategy. -[role=query-example] +[role=query-example, no-result=true] -- .The following will run the algorithm with sampled strategy and stream results: [source, cypher, role=noplay] ---- -CALL gds.wcc.stream('myIndexedGraph') +CALL gds.wcc.stream('myIndexedGraph', {concurrency: 1, consecutiveIds: true}) YIELD nodeId, componentId RETURN gds.util.asNode(nodeId).name AS name, componentId ORDER BY componentId, name @@ -644,11 +644,15 @@ ORDER BY componentId, name | "Alice" | 0 | "Bridget" | 0 | "Charles" | 0 -| "Doug" | 3 -| "Mark" | 3 -| "Michael" | 3 +| "Doug" | 1 +| "Mark" | 1 +| "Michael" | 1 |=== -- -As expected, the results are the same as before. -However, on larger graphs, there will be a notable improvement in compute time. +[NOTE] +==== +Because of the randomness in the Graph sampling optimization we are using `concurrency: 1` in the example. +Running it with higher concurrency may yield different componentIds but the actual components should stay the same. +For our case here it would be equally plausible to get the inverse solution, f.i. when our community `0` nodes are mapped to community `3` instead, and vice versa. +==== diff --git a/doc/modules/ROOT/pages/algorithms/yens.adoc b/doc/modules/ROOT/pages/algorithms/yens.adoc index c5bfac979ed..8e5f9a37af8 100644 --- a/doc/modules/ROOT/pages/algorithms/yens.adoc +++ b/doc/modules/ROOT/pages/algorithms/yens.adoc @@ -1,11 +1,12 @@ [[algorithms-yens]] -= Yen's algorithm Shortest Path += Yen's Shortest Path algorithm :description: This section describes the Yen's Shortest Path algorithm in the Neo4j Graph Data Science library. :entity: source-target-pair :result: shortest path :algorithm: Yen's :source-target: true :procedure-name: pass:q[gds.shortestPath.yens] +:sequential: true :directed: diff --git a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc index 72e7f6f27fb..00f98c46673 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc @@ -65,7 +65,7 @@ YIELD startNodeId, targetNodeId, distance image::example-graphs/shortest-path_graph.png[] .The following will create a sample graph: -[source, cypher, role=noplay] +[source, cypher, role=noplay setup-query] ---- CREATE (a:Loc {name: 'A'}), (b:Loc {name: 'B'}), @@ -85,8 +85,10 @@ CREATE (a:Loc {name: 'A'}), ---- +=== Using native projection + .The following will project and store a graph using native projection: -[source, cypher, role=noplay] +[source, cypher, role=noplay graph-project-query, group=native] ---- CALL gds.graph.project( 'nativeGraph', @@ -100,8 +102,10 @@ CALL gds.graph.project( YIELD graphName ---- +[role=query-example] +-- .The following will run the algorithm and stream results: -[source, cypher, role=noplay] +[source, cypher, role=noplay, group=native] ---- CALL gds.alpha.allShortestPaths.stream('nativeGraph', { relationshipWeightProperty: 'cost' @@ -122,42 +126,43 @@ LIMIT 10 .Results [opts="header",cols="1,1,1"] |=== -| Source | Target | Cost -| A | F | 160 -| A | E | 120 -| B | F | 110 -| C | F | 110 -| A | D | 90 -| B | E | 70 -| C | E | 70 -| D | F | 70 -| A | B | 50 -| A | C | 50 +| source | target | distance +| "A" | "F" | 160 +| "A" | "E" | 120 +| "B" | "F" | 110 +| "C" | "F" | 110 +| "A" | "D" | 90 +| "B" | "E" | 70 +| "C" | "E" | 70 +| "D" | "F" | 70 +| "A" | "B" | 50 +| "A" | "C" | 50 |=== +-- This query returned the top 10 pairs of nodes that are the furthest away from each other. F and E appear to be quite distant from the others. -For now, only single-source shortest path support loading the relationship as undirected, but we can use Cypher loading to help us solve this. -Undirected graph can be represented as https://en.wikipedia.org/wiki/Bidirected_graph[Bidirected graph], which is a directed graph in which the reverse of every relationship is also a relationship. - -We do not have to save this reversed relationship, we can project it using *Cypher loading*. -Note that relationship query does not specify direction of the relationship. -This is applicable to all other algorithms that use Cypher loading. +=== Using Cypher aggregation -.The following will project and store an undirected graph using cypher projection: -[source, cypher, role=noplay] +.The following will project and store an undirected graph using cypher aggregation: +[source, cypher, role=noplay graph-project-query, group=cypher] ---- -CALL gds.graph.project.cypher( +MATCH (src:Loc)-[r:ROAD]->(trg:Loc) +RETURN gds.alpha.graph.project( 'cypherGraph', - 'MATCH (n:Loc) RETURN id(n) AS id', - 'MATCH (n:Loc)-[r:ROAD]-(p:Loc) RETURN id(n) AS source, id(p) AS target, r.cost AS cost' -) -YIELD graphName + src, + trg, + {}, + {relationshipType: type(r), properties: {cost: r.cost}}, + {undirectedRelationshipTypes: ['ROAD'] +}) ---- +[role=query-example] +-- .The following will run the algorithm, treating the graph as undirected: -[source, cypher, role=noplay] +[source, cypher, role=noplay, group=cypher] ---- CALL gds.alpha.allShortestPaths.stream('cypherGraph', { relationshipWeightProperty: 'cost' @@ -178,15 +183,16 @@ LIMIT 10 .Results [opts="header",cols="1,1,1"] |=== -| Source | Target | Cost -| A | F | 160 -| F | A | 160 -| A | E | 120 -| E | A | 120 -| B | F | 110 -| C | F | 110 -| F | B | 110 -| F | C | 110 -| A | D | 90 -| D | A | 90 +| source | target | distance +| "A" | "F" | 160 +| "F" | "A" | 160 +| "A" | "E" | 120 +| "E" | "A" | 120 +| "B" | "F" | 110 +| "C" | "F" | 110 +| "F" | "B" | 110 +| "F" | "C" | 110 +| "A" | "D" | 90 +| "D" | "A" | 90 |=== +-- diff --git a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc index 7269360fd17..41ee86f71cd 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc @@ -3,24 +3,38 @@ = Minimum Weight k-Spanning Tree :description: This section describes the Minimum Weight k-Spanning Tree algorithm in the Neo4j Graph Data Science library. :entity: node -:result: spanning tree edge +:result: spanning tree :algorithm: k-Spanning Tree heuristic +:sequential: true include::partial$/operations-reference/alpha-note.adoc[] == Introduction -Sometimes, we want to limit the size of our spanning tree result, as we are only interested in finding a smaller tree within the graph, and not one that necessarily spans across all nodes. +Sometimes, we might require a spanning tree(a tree where its nodes are connected with each via a single path) that does not necessarily span all nodes in the graph. The K-Spanning tree heuristic algorithm returns a tree with `k` nodes and `k − 1` relationships. -Our heuristic processes the result found by the Prim algorithm for the Minimum Weight Spanning Tree problem. +Our heuristic processes the result found by Prim's algorithm for the xref:algorithms/minimum-weight-spanning-tree.adoc[Minimum Weight Spanning Tree] problem. +Like Prim, it starts from a given source node, finds a spanning tree for all nodes and then removes nodes using heuristics to produce a tree with 'k' nodes. +Note that the source node will not be necessarily included in the final output as the heuristic tries to find a globally good tree. [[algorithms-k-spanning]] == Considerations -The minimum weight k-Spanning Tree is NP-Hard. The algorithm in the Neo4j GDS Library is therefore not guaranteed to find the optimal answer, but should return good approximation in practice. +The Minimum weight k-Spanning Tree is NP-Hard. The algorithm in the Neo4j GDS Library is therefore not guaranteed to find the optimal answer, but should hopefully return a good approximation in practice. + +Like Prim algorithm, the algorithm focuses only on the component of the source node. If that component has fewer than `k` nodes, it will not look into other components, but will instead return the component. + [[algorithms-minimum-k-weight-spanning-tree-syntax]] == Syntax +include::partial$/algorithms/shared/syntax-intro-named-graph.adoc[] + +.K Spanning Tree syntax per mode +[.tabbed-example, caption = ] +==== + +[.include-with-write] +====== .The following will run the k-spanning tree algorithms and write back results: [source, cypher, role=noplay] ---- @@ -31,19 +45,19 @@ CALL gds.alpha.kSpanningTree.write( YIELD effectiveNodeCount: Integer, preProcessingMillis: Integer, computeMillis: Integer, + postProcessingMillis: Integer, writeMillis: Integer, configuration: Map ---- +include::partial$/algorithms/common-configuration/common-parameters.adoc[] .Configuration -[opts="header",cols="1,1,1,1,4"] +[opts="header",cols="3,2,3m,2,8"] |=== -| Name | Type | Default | Optional | Description -| k | Integer | null | no | The result is a tree with `k` nodes and `k − 1` relationships. -| sourceNode | Integer | null | no | The start node ID. -| xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. -| writeProperty | String | n/a | no | The partition that a node belongs to. -| objective | String | 'minimum' | yes | If specified, the parameter dictates whether to try and find the minimum or the maximum k-spanning tree. By default, the algorithm looks for the minimum one. Permitted values are 'minimum' and 'maximum'. +| Name | Type | Default | Optional | Description +include::partial$/algorithms/common-configuration/common-write-configuration-entries.adoc[] +include::partial$/algorithms/k-spanning-tree/specific-configuration.adoc[] + |=== .Results @@ -53,16 +67,24 @@ YIELD effectiveNodeCount: Integer, | effectiveNodeCount | Integer | The number of visited nodes. | preProcessingMillis | Integer | Milliseconds for preprocessing the data. | computeMillis | Integer | Milliseconds for running the algorithm. +| postProcessingMillis | Integer | Milliseconds for postprocessing results of the algorithm. | writeMillis | Integer | Milliseconds for writing result data back. +| configuration | Map | The configuration used for running the algorithm. + |=== +====== +==== [[algorithms-minimum-weight-spanning-tree-sample]] == Minimum Weight k-Spanning Tree algorithm examples -image::mst.png[] +:algorithm-name: {algorithm} +:graph-description: road network +:image-file: spanning-tree-graph.svg +include::partial$/algorithms/shared/examples-intro.adoc[] .The following will create the sample graph depicted in the figure: -[source, cypher, role=noplay] +[source, cypher, role=noplay setup-query] ---- CREATE (a:Place {id: 'A'}), (b:Place {id: 'B'}), @@ -81,7 +103,7 @@ CREATE (a:Place {id: 'A'}), ---- .The following will project and store a named graph: -[source, cypher, role=noplay] +[source, cypher, role=noplay graph-project-query] ---- CALL gds.graph.project( 'graph', @@ -98,46 +120,57 @@ CALL gds.graph.project( [[algorithms-minimum-weight-spanning-tree-k]] == K-Spanning tree examples -In our sample graph we have 5 nodes. -When we ran MST above, we got a 5-minimum spanning tree returned, that covered all five nodes. -By setting the `k=3`, we define that we want to get returned a 3-minimum spanning tree that covers 3 nodes and has 2 relationships. +=== Minimum K-Spanning Tree example + +In our sample graph we have 7 nodes. +By setting the `k=3`, we define that we want to find a 3-minimum spanning tree that covers 3 nodes and has 2 relationships. .The following will run the k-minimum spanning tree algorithm and write back results: +[role=query-example, no-result=true, group=write-example] +-- [source, cypher, role=noplay] ---- -MATCH (n:Place{id: 'D'}) +MATCH (n:Place{id: 'A'}) CALL gds.alpha.kSpanningTree.write('graph', { k: 3, sourceNode: id(n), relationshipWeightProperty: 'cost', - writeProperty:'kminst' + writeProperty:'kmin' }) YIELD preProcessingMillis, computeMillis, writeMillis, effectiveNodeCount RETURN preProcessingMillis,computeMillis,writeMillis, effectiveNodeCount; ---- +-- -.Find nodes that belong to our k-spanning tree result: +[role=query-example, group=write-example] +-- +.The following will find the nodes that belong to our k-spanning tree result: [source, cypher, role=noplay] ---- -MATCH (n:Place) -WITH n.id AS Place, n.kminst AS Partition, count(*) AS count -WHERE count = 3 -RETURN Place, Partition +MATCH (n) +WITH n.kmin AS p, count(n) AS c +WHERE c = 3 +MATCH (n) +WHERE n.kmin = p +RETURN n.id As Place, p as Partition + ---- .Results [opts="header",cols="1,1"] |=== | Place | Partition -| A | 1 -| B | 1 -| C | 1 -| D | 3 -| E | 4 +| "A" | 0 +| "B" | 0 +| "C" | 0 |=== +-- +Nodes A, B, and C form the discovered 3-minimum spanning tree of our graph. -Nodes A, B, and C are the result 3-minimum spanning tree of our graph. +=== Maximum K-Spanning Tree example +[role=query-example,no-result=true, group=max-example] +-- .The following will run the k-maximum spanning tree algorithm and write back results: [source, cypher, role=noplay] ---- @@ -146,31 +179,33 @@ CALL gds.alpha.kSpanningTree.write('graph', { k: 3, sourceNode: id(n), relationshipWeightProperty: 'cost', - writeProperty:'kmaxst', - objective: 'maximum', + writeProperty:'kmax', + objective: 'maximum' }) YIELD preProcessingMillis, computeMillis, writeMillis, effectiveNodeCount RETURN preProcessingMillis,computeMillis,writeMillis, effectiveNodeCount; ---- +-- +[role=query-example, group=max-example] +-- .Find nodes that belong to our k-spanning tree result: [source, cypher, role=noplay] ---- -MATCH (n:Place) -WITH n.id AS Place, n.kmaxst AS Partition, count(*) AS count -WHERE count = 3 -RETURN Place, Partition +MATCH (n) +WITH n.kmax AS p, count(n) AS c +WHERE c = 3 +MATCH (n) +WHERE n.kmax = p +RETURN n.id As Place, p as Partition ---- - .Results [opts="header",cols="1,1"] |=== | Place | Partition -| A | 0 -| B | 1 -| C | 3 -| D | 3 -| E | 3 +| "C" | 3 +| "D" | 3 +| "E" | 3 |=== - -Nodes C, D, and E are the result 3-maximum spanning tree of our graph. +-- +Nodes C, D, and E form a 3-maximum spanning tree of our graph. diff --git a/doc/modules/ROOT/pages/common-usage/running-algos.adoc b/doc/modules/ROOT/pages/common-usage/running-algos.adoc index 456e4040e0c..4692cc97412 100644 --- a/doc/modules/ROOT/pages/common-usage/running-algos.adoc +++ b/doc/modules/ROOT/pages/common-usage/running-algos.adoc @@ -130,3 +130,7 @@ The Default is `concurrency` [[common-configuration-jobid]] jobId - String:: An id for the job to be started can be provided in order for it to be more easily tracked with eg. GDS's xref:common-usage/logging.adoc[logging capabilities]. + +[[common-configuration-logProgress]] +logProgress - Boolean:: +Configuration parameter that allows to turn `off/on` percentage logging while running procedure. It is `on` by default diff --git a/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc b/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc index bfa36e83220..1e1d53454b8 100644 --- a/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc +++ b/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc @@ -661,6 +661,7 @@ ORDER BY source ASC, target ASC NOTE: The properties we want to stream must exist for each specified relationship type. +[[catalog-graph-relationship-to-undirected-example]] === Convert to undirected Some algorithms such as Triangle Count and Link Prediction expect undirected relationships. The following shows how to convert the relationships of type `LIKES` in the graph from directed to undirected by creating an undirected relationship of new type `INTERACTS`. @@ -684,7 +685,7 @@ YIELD [opts="header"] |=== | inputRelationships | relationshipsWritten -| 19 | 18 +| 9 | 18 |=== -- diff --git a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc index 6db24ce6a99..fa8fa6d515b 100644 --- a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc +++ b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc @@ -3,7 +3,6 @@ = Projecting graphs using Apache Arrow :description: This chapter explains how to import data using Apache Arrow™ into the Graph Data Science library. - include::partial$/operations-reference/alpha-note.adoc[] Projecting graphs via https://arrow.apache.org/[Apache Arrow] allows importing graph data which is stored outside of Neo4j. @@ -55,26 +54,25 @@ See xref:graph-project-apache-arrow.adoc#arrow-send-relationships[Sending relati [[arrow-initialize-import-process]] -== Initializing the Import Process +== Initializing the import process An import process is initialized by sending a Flight action using the action type `CREATE_GRAPH`. The action body is a JSON document containing metadata for the import process: ---- { - name: "my_graph", - database_name: "neo4j", - concurrency: 4, - undirected_relationship_types: [] - inverse_indexed_relationship_types: [] + name: "my_graph", <1> + database_name: "neo4j", <2> + concurrency: 4, <3> + undirected_relationship_types: [] <4> + inverse_indexed_relationship_types: [] <5> } ---- - -The `name` is used to identify the import process, it is also the name of the resulting in-memory graph in the graph catalog. -The `database_name` is used to tell the server on which database the projected graph will be available. -The `concurrency` key is optional, it is used during finalizing the in-memory graph on the server after all data has been received. -The `undirected_relationship_types` key is optional, it is used to declare a number of relationship types as undirected. Relationships with the specified types will be imported as undirected. `*` can be used to declare all relationship types as undirected. -The `inverse_indexed_relationship_types` key is optional, it is used to declare a number of relationship types which will also be indexed in inverse direction. `*` can be used to declare all relationship types as inverse indexed. +<1> Used to identify the import process. It is also the name of the resulting in-memory graph in the graph catalog. +<2> The name of the database on which the projected graph will be available. +<3> (optional) The level of concurrency that will be set on the in-memory graph after all data has been received. +<4> (optional) A list of relationship types that must be imported as undirected. A wildcard (`*`) can be used to include all the types. +<5> (optional) A list of relationship types that must be indexed in inverse direction. A wildcard (`*`) can be used to include all the types. [NOTE] Relationships declared as undirected should only be provided once, i.e. in a single direction. @@ -235,6 +233,8 @@ The timeout can be configured via the `gds.arrow.abortion_timeout` setting, for == Creating a Neo4j database +include::partial$/common-usage/not-on-aurads-note.adoc[] + The xref:graph-project-apache-arrow.adoc#arrow-client-server-protocol[Client-Server protocol] can also be used to create a new Neo4j database instead of an in-memory graph. To initialize a database import process, we need to change the initial action type to `CREATE_DATABASE`. The action body is a JSON document containing the configuration for the import process: diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index fa727199f02..9df47aa81a7 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -1,11 +1,9 @@ [[System-requirements]] = System Requirements -:neo4j-docs-link-version: 4.4 - == Main Memory -The GDS library runs within a Neo4j instance and is therefore subject to the general https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/performance/memory-configuration/[Neo4j memory configuration]. +The GDS library runs within a Neo4j instance and is therefore subject to the general https://neo4j.com/docs/operations-manual/4.4/performance/memory-configuration/[Neo4j memory configuration]. .GDS heap memory usage image::memory-usage.png[width=600] @@ -14,10 +12,25 @@ image::memory-usage.png[width=600] [[heap-size]] === Heap size -The heap space is used for storing graph projections in the graph catalog and algorithm state. -When writing algorithm results back to Neo4j, heap space is also used for handling transaction state (see https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.tx_state.memory_allocation[dbms.tx_state.memory_allocation]). +The heap space is used for storing graph projections in the graph catalog, and algorithm state. + +[.tabbed-example, caption = ] +==== + +[.include-with-neo4j-5x] +===== +When writing algorithm results back to Neo4j, heap space is also used for handling transaction state (see https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#config_db.tx_state.memory_allocation[dbms.tx_state.memory_allocation]). +For purely analytical workloads, a general recommendation is to set the heap space to about 90% of the available main memory. +This can be done via https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#config_server.memory.heap.initial_size[server.memory.heap.initial_size] and https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#config_server.memory.heap.max_size[server.memory.heap.max_size]. +===== + +[.include-with-neo4j-4x] +===== +When writing algorithm results back to Neo4j, heap space is also used for handling transaction state (see https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#config_dbms.tx_state.memory_allocation[dbms.tx_state.memory_allocation]). For purely analytical workloads, a general recommendation is to set the heap space to about 90% of the available main memory. -This can be done via https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.heap.initial_size[dbms.memory.heap.initial_size] and https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.heap.max_size[dbms.memory.heap.max_size]. +This can be done via https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#config_dbms.memory.heap.initial_size[dbms.memory.heap.initial_size] and https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#config_dbms.memory.heap.max_size[dbms.memory.heap.max_size]. +===== +==== To better estimate the heap space required to project graphs and run algorithms, consider the xref:common-usage/memory-estimation.adoc[Memory Estimation] feature. The feature estimates the memory consumption of all involved data structures using information about number of nodes and relationships from the Neo4j count store. @@ -26,7 +39,24 @@ The feature estimates the memory consumption of all involved data structures usi The page cache is used to cache the Neo4j data and will help to avoid costly disk access. -For purely analytical workloads including xref:management-ops/projections/graph-project.adoc[native projections], it is recommended to decrease https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.pagecache.size[dbms.memory.pagecache.size] in favor of an increased heap size. +For purely analytical workloads including xref:management-ops/projections/graph-project.adoc[native projections], it is recommended to decrease the configured PageCache in favor of an increased heap size. + +[.tabbed-example, caption = ] +==== + +[.include-with-neo4j-5x] +===== +To configure the PageCache size you can use the following Neo4j configuration property +https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#config_server.memory.pagecache.size[server.memory.pagecache.size] +===== + +[.include-with-neo4j-4x] +===== +To configure the PageCache size you can use the following Neo4j configuration property +https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#config_dbms.memory.pagecache.size[dbms.memory.pagecache.size] +===== +==== + However, setting a minimum page cache size is still important when projecting graphs: * For xref:management-ops/projections/graph-project.adoc[native projections], the minimum page cache size for projecting a graph can be roughly estimated by `8KB * 100 * readConcurrency`. @@ -39,7 +69,7 @@ Ideally, if the xref:common-usage/memory-estimation.adoc[memory estimation] feat [NOTE] ==== Decreasing the page cache size in favor of heap size is *not* recommended if the Neo4j instance runs both, operational and analytical workloads at the same time. -See https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/performance/memory-configuration/[Neo4j memory configuration] for general information about page cache sizing. +See https://neo4j.com/docs/operations-manual/4.4/performance/memory-configuration/[Neo4j memory configuration] for general information about page cache sizing. ==== [[system-requirements-cpu]] @@ -58,6 +88,6 @@ The maximum concurrency that can be used is limited depending on the license und * Neo4j Graph Data Science Library - Enterprise Edition (GDS EE) ** The maximum concurrency in the library is unlimited. -To register for a license, please contact Neo4j at https://neo4j.com/contact-us/?ref=graph-data-science. +To register for a license, please visit https://neo4j.com/contact-us/?ref=graph-data-science[neo4j.com]. NOTE: Concurrency limits are determined based on whether you have a GDS EE license, or if you are using GDS CE. The maximum concurrency limit in the graph data science library is not set based on your edition of the Neo4j database. diff --git a/doc/modules/ROOT/pages/installation/index.adoc b/doc/modules/ROOT/pages/installation/index.adoc index cc36aa9364a..84e9a30df48 100644 --- a/doc/modules/ROOT/pages/installation/index.adoc +++ b/doc/modules/ROOT/pages/installation/index.adoc @@ -5,17 +5,14 @@ The Neo4j Graph Data Science (GDS) library is delivered as a plugin to the Neo4j Graph Database. The plugin needs to be installed into the database and added to the allowlist in the Neo4j configuration. -There are two main ways of achieving this, which we will detail in this chapter. - - -This chapter is divided into the following sections: +The following sections cover the different installation methods: . xref:installation/supported-neo4j-versions.adoc[Supported Neo4j versions] . xref:installation/neo4j-desktop.adoc[Neo4j Desktop] . xref:installation/neo4j-server.adoc[Neo4j Server] . xref:installation/installation-enterprise-edition.adoc[Enterprise Edition Configuration] . xref:installation/installation-docker.adoc[Neo4j Docker] -. xref:installation/installation-causal-cluster.adoc[Neo4j Causal Cluster] +. xref:installation/installation-neo4j-cluster.adoc[Neo4j cluster] . xref:installation/installation-apache-arrow.adoc[Apache Arrow] . xref:installation/additional-config-parameters.adoc[Additional configuration options] . xref:installation/System-requirements.adoc[System Requirements] diff --git a/doc/modules/ROOT/pages/installation/installation-causal-cluster.adoc b/doc/modules/ROOT/pages/installation/installation-causal-cluster.adoc deleted file mode 100644 index c78fee8fad1..00000000000 --- a/doc/modules/ROOT/pages/installation/installation-causal-cluster.adoc +++ /dev/null @@ -1,11 +0,0 @@ -[[installation-causal-cluster]] -= Neo4j Causal Cluster - -include::partial$/common-usage/not-on-aurads-note.adoc[] - -In a Neo4j Causal Cluster, GDS should only be installed on a server that is not essential for handling transactional load. This is because the compute-intensive OLAP workloads in GDS might interfere with the smooth operation of the OLTP system that is Neo4j Causal Cluster. - -In order to install the GDS library you can follow the steps from xref:installation/neo4j-server.adoc[Neo4j Server]. -Additionally, the Neo4j Causal Cluster must be configured to use https://neo4j.com/docs/operations-manual/current/clustering/internals/#causal-clustering-routing[server-side routing]. - -For more details, see xref:production-deployment/causal-cluster.adoc[GDS with Neo4j Causal Cluster]. diff --git a/doc/modules/ROOT/pages/installation/installation-enterprise-edition.adoc b/doc/modules/ROOT/pages/installation/installation-enterprise-edition.adoc index 232d9245c82..356461f0574 100644 --- a/doc/modules/ROOT/pages/installation/installation-enterprise-edition.adoc +++ b/doc/modules/ROOT/pages/installation/installation-enterprise-edition.adoc @@ -2,7 +2,7 @@ = Enterprise Edition Configuration Unlocking the Enterprise Edition of the Neo4j Graph Data Science library requires a valid license key. -To register for a license, please contact Neo4j at https://neo4j.com/contact-us/?ref=graph-analytics. +To register for a license, please visit https://neo4j.com/contact-us/?ref=graph-data-science[neo4j.com]. The license is issued in the form of a license key file, which needs to be placed in a directory accessible by the Neo4j server. You can configure the location of the license key file by setting the `gds.enterprise.license_file` option in the `neo4j.conf` configuration file of your Neo4j installation. diff --git a/doc/modules/ROOT/pages/installation/installation-neo4j-cluster.adoc b/doc/modules/ROOT/pages/installation/installation-neo4j-cluster.adoc new file mode 100644 index 00000000000..cef70ad9eea --- /dev/null +++ b/doc/modules/ROOT/pages/installation/installation-neo4j-cluster.adoc @@ -0,0 +1,11 @@ +[[installation-neo4j-cluster]] += Neo4j cluster + +include::partial$/common-usage/not-on-aurads-note.adoc[] + +In a Neo4j cluster, GDS should only be installed on a server that is not essential for handling transactional load. This is because the compute-intensive OLAP workloads in GDS might interfere with the smooth operation of the OLTP system that is Neo4j cluster. + +In order to install the GDS library you can follow the steps from xref:installation/neo4j-server.adoc[Neo4j Server]. +Additionally, the Neo4j cluster must be configured to use https://neo4j.com/docs/operations-manual/current/clustering/setup/routing/#clustering-routing[server-side routing]. + +For more details, see xref:production-deployment/neo4j-cluster.adoc[GDS with Neo4j cluster]. diff --git a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc index fbbe1cc8ceb..21265eae603 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc @@ -20,5 +20,3 @@ If the procedure allowlist is configured, make sure to also include procedures f ---- dbms.security.procedures.allowlist=gds.* ---- - -NOTE: Before `Neo4j 4.2`, the configuration setting is called `dbms.security.procedures.whitelist` diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index 0c946584f91..fcc0af6aec3 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -1,20 +1,13 @@ [[neo4j-server]] = Neo4j Server -The GDS library is intended to be used on a standalone Neo4j server. +On a standalone Neo4j Server, GDS will need to be installed and configured manually. -[NOTE] -==== -Running the GDS library on a core member of a Neo4j Causal Cluster is not supported. -Read more about how to use GDS in conjunction with Neo4j Causal Cluster deployment xref:installation/installation-causal-cluster.adoc[below]. -==== +1. Download `neo4j-graph-data-science-[version].zip` from the https://neo4j.com/download-center/#ngds[Neo4j Download Center] -On a standalone Neo4j Server, the library will need to be installed and configured manually. +2. Unzip the archive and move the `neo4j-graph-data-science-[version].jar` file into the `$NEO4J_HOME/plugins` directory. -1. Download `neo4j-graph-data-science-[version].jar` from the https://neo4j.com/download-center/#algorithms[Neo4j Download Center] and copy it into the `$NEO4J_HOME/plugins` directory. - - -2. Add the following to your `$NEO4J_HOME/conf/neo4j.conf` file: +3. Add the following to your `$NEO4J_HOME/conf/neo4j.conf` file: + ---- dbms.security.procedures.unrestricted=gds.* @@ -22,32 +15,29 @@ dbms.security.procedures.unrestricted=gds.* This configuration entry is necessary because the GDS library accesses low-level components of Neo4j to maximise performance. + -3. Check if the procedure allowlist is enabled in the `$NEO4J_HOME/conf/neo4j.conf` file and add the GDS library if necessary: +4. Check if the procedure allowlist is enabled in the `$NEO4J_HOME/conf/neo4j.conf` file and add the GDS library if necessary: + ---- dbms.security.procedures.allowlist=gds.* ---- + -NOTE: Before `Neo4j 4.2`, the configuration setting is called `dbms.security.procedures.whitelist` - - -4. Restart Neo4j +5. Restart Neo4j [[neo4j-server-verify]] == Verifying installation -To verify your installation, the library version can be printed by entering into the browser in Neo4j Desktop and calling the `gds.version()` function: +To verify your installation, print the version of Graph Data Science by opening Neo4j Browser and running the `gds.version()` function: [source, cypher, role=noplay] ---- -RETURN gds.version() +RETURN gds.version(); ---- -To list all installed algorithms, run the `gds.list()` procedure: +To list all available procedures, run the `gds.list()` procedure: [source, cypher, role=noplay] ---- -CALL gds.list() +CALL gds.list(); ---- diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 562ac0194ea..ef221582a62 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -4,22 +4,19 @@ Below is the compatibility matrix for the GDS library vs Neo4j. In general, you can count on the latest version of GDS supporting the latest version of Neo4j and vice versa, and we recommend you always upgrade to that combination. -We list software with major and minor version only, e.g. GDS library 1.8. -You should read that as any patch version of that major+minor version, but again, do upgrade to the latest patch always, to ensure you get all bug fixes included. - Not finding your version of GDS or Neo4j listed? Time to upgrade! [opts=header] |=== -| Neo4j Graph Data Science | Neo4j version -.3+<.^|`2.3` -| `5.1` -| `5.2` -| `4.4`, at least `4.4.9` -.4+<.^|`2.2` -| `5.1` -| `5.2` -| `4.4`, at least `4.4.9` -| `4.3`, at least `4.3.15` +| Neo4j version | Neo4j Graph Data Science +| `5.8` | `2.3.6 or later` +| `5.7` | `2.3.3 or later` +| `5.6` | `2.3.2 or later` +| `5.5` | `2.3.1 or later` +| `5.4` | `2.3`, `2.2.7 or later` footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] +| `5.3` | `2.3`, `2.2.6 or later` footnote:eol[] +| `5.2` | `2.3`, `2.2.3 or later` footnote:eol[] +| `5.1` | `2.3`, `2.2.1` footnote:eol[] +| `4.4.9 or later` | `2.3`, `2.2` footnote:eol[] |=== diff --git a/doc/modules/ROOT/pages/introduction.adoc b/doc/modules/ROOT/pages/introduction.adoc index 93a7496f956..ab7e071502c 100644 --- a/doc/modules/ROOT/pages/introduction.adoc +++ b/doc/modules/ROOT/pages/introduction.adoc @@ -1,7 +1,7 @@ [[introduction]] = Introduction :description: This chapter provides a brief introduction of the main concepts in the Neo4j Graph Data Science library. - +:keywords: alpha, beta, Production-quality, api tiers The Neo4j Graph Data Science (GDS) library provides efficiently implemented, parallel versions of common graph algorithms, exposed as Cypher procedures. Additionally, GDS includes machine learning pipelines to train predictive supervised models to solve graph problems, such as predicting missing relationships. @@ -83,30 +83,4 @@ The amount of data loaded can be controlled by so called graph projections, whic For more information see xref:management-ops/index.adoc[Graph Management]. - -[[introduction-editions]] -== Editions - -The Neo4j Graph Data Science library is available in two editions. - -* The open source Community Edition: -** Includes all algorithms. -** Most of the catalog operations to manage graphs, models and pipelines are available. Unavailable operations are listed below. -** Limits the concurrency to 4 CPU cores. -** Limits the capacity of the model catalog to 4 models. -* The Neo4j Graph Data Science library Enterprise Edition: -** Can run on an unlimited amount of CPU cores. -** Supports the role-based access control system (RBAC) from Neo4j Enterprise Edition. -** Support running GDS as part of a xref::production-deployment/causal-cluster.adoc[cluster deployment]. -** Includes capacity and load xref::common-usage/monitoring-system.adoc[monitoring]. -** Supports various additional graph catalog features, including: -*** Graph xref::management-ops/backup-restore.adoc[backup and restore]. -*** Data import and export via xref:installation/installation-apache-arrow.adoc[Apache Arrow]. -** Supports various additional model catalog features, including: -*** Storing unlimited amounts of models in the model catalog. -*** Sharing of models between users, by xref:model-catalog/publish.adoc[publishing it]. -*** Model xref:model-catalog/store.adoc#model-catalog-store-ops[persistence to disk]. -** Supports an xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[optimized graph implementation]. -** Allows the configuration of xref:management-ops/defaults-and-limits.adoc[defaults and limits]. - -For more information see xref:installation/System-requirements.adoc#system-requirements-cpu[System Requirements - CPU]. +include::partial$/introduction/enterprise-features.adoc[leveloffset=+1] diff --git a/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc b/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc index 3ec6eb7e72c..d81ceec1a63 100644 --- a/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc +++ b/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc @@ -2,6 +2,12 @@ = Auto-tuning :description: This section describes auto-tuning for hyper-parameters in training pipelines in the Neo4j Graph Data Science library. +[TIP] +==== +Auto-tuning is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning] +==== xref:machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc[Node Classification Pipelines], xref:machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc[Node Regression Pipelines], and xref:machine-learning/linkprediction-pipelines/link-prediction.adoc[Link Prediction Pipelines] are trained using supervised machine learning methods which have multiple configurable parameters that affect training outcomes. To obtain models with high quality, setting good values for the hyper-parameters can have a large impact. diff --git a/doc/modules/ROOT/pages/machine-learning/linkprediction-pipelines/training.adoc b/doc/modules/ROOT/pages/machine-learning/linkprediction-pipelines/training.adoc index ffe836b2107..eb1373392d0 100644 --- a/doc/modules/ROOT/pages/machine-learning/linkprediction-pipelines/training.adoc +++ b/doc/modules/ROOT/pages/machine-learning/linkprediction-pipelines/training.adoc @@ -312,7 +312,7 @@ CALL gds.graph.project( 'fullGraph', { Person: { - properties: ['age'] + properties: {age: {defaultValue: 1}} }, City: { properties: {age: {defaultValue: 1}} diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc index 6dc3b2c7256..ae7f6bee0fe 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc @@ -5,13 +5,20 @@ :result: embedding :algorithm: FastRP - :directed: :undirected: :homogeneous: :weighted: include::partial$/algorithms/shared/algorithm-traits.adoc[] +{nbsp} + +[TIP] +==== +FastRP is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example] +==== + [[algorithms-embeddings-fastrp-introduction]] == Introduction @@ -58,7 +65,7 @@ In general, it is recommended to first use `UNDIRECTED` as this is what the orig For more information on this algorithm see: * https://arxiv.org/pdf/1908.11512.pdf[H. Chen, S.F. Sultan, Y. Tian, M. Chen, S. Skiena: Fast and Accurate Network Embeddings via Very Sparse Random Projection, 2019.^] -* https://core.ac.uk/download/pdf/82724427.pdf[Dimitris Achlioptas. Database-friendly random projections: Johnson-Lindenstrauss with binary coins. Journal of Computer and System Sciences, 66(4):671–687, 2003.] +* https://www.sciencedirect.com/science/article/pii/S0022000003000254[Dimitris Achlioptas. Database-friendly random projections: Johnson-Lindenstrauss with binary coins. Journal of Computer and System Sciences, 66(4):671–687, 2003.] === Node properties diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc index 7f81c636c87..08b2e7ec7ec 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc @@ -12,6 +12,13 @@ include::partial$/operations-reference/beta-note.adoc[] :heterogeneous: include::partial$/algorithms/shared/algorithm-traits.adoc[] +{nbsp} + +[TIP] +==== +HashGNN is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning] +==== [[algorithms-embeddings-hashgnn-introduction]] == Introduction @@ -238,6 +245,22 @@ This is especially true for unsupervised embedding algorithms such as HashGNN. Therefore, caution should be taken when using many iterations in the heterogeneous mode. +=== Random seed + +The random seed has a special role in this algorithm. +Other than making all steps of the algorithm deterministic, the `randomSeed` parameter determines which (to some degree) hash functions are used inside the algorithm. +This is important since it greatly affects which features are sampled each iteration. +The hashing plays a similar role to the (typically neural) transformations in each layer of Graph Neural Networks, which tells us something about how important the hash functions are. +Indeed, one can often see a significant difference in the quality of the node embeddings output from the algorithm when only the `randomSeed` is different in the configuration. + +For these reasons it can actually make sense to tune the random seed parameter. +Note that it should be tuned as a categorical (i.e. non-ordinal) number, meaning that values 1 and 2 can be considered just as similar or different as 1 and 100. +A good way to start doing this is to choose 5 - 10 arbitrary integers (eg. values 1, 2, 3, 4 and 5) as the candidates for the random seed. + +`randomSeed` codepends on several configuration parameters, and in particular on the `neighborInfluence` parameter which also directly influences which hash functions are used. +Therefore, if `neighborInfluence` is changed, likely the `randomSeed` parameter needs to be retuned. + + [[algorithms-embeddings-hashgnn-syntax]] == Syntax @@ -354,7 +377,7 @@ CREATE (matt)-[:LIKES]->(mango), (jeff)-[:LIKES]->(mango), (brie)-[:LIKES]->(banana), - (else)-[:LIKES]->(plum), + (elsa)-[:LIKES]->(plum), (john)-[:LIKES]->(plum), (dan)-[:KNOWS]->(annie), @@ -434,7 +457,7 @@ YIELD nodeCount, relationshipCount, bytesMin, bytesMax, requiredMemory [opts="header", cols="1,1,1,1,1"] |=== | nodeCount | relationshipCount | bytesMin | bytesMax | requiredMemory -| 7 | 18 | 59160 | 59160 | "57 KiB" +| 7 | 18 | 2040 | 2040 | "2040 Bytes" |=== -- @@ -574,17 +597,17 @@ YIELD nodeId, embedding .Results |=== | nodeId | embedding -| 0 | [1.0, 1.0, 0.0, 0.0, 0.0, 0.0] -| 1 | [1.0, 1.0, 0.0, 0.0, 0.0, 0.0] -| 2 | [1.0, 0.0, 1.0, 0.0, 0.0, 0.0] +| 0 | [1.0, 1.0, 0.0, 0.0, 0.0, 1.0] +| 1 | [1.0, 1.0, 1.0, 0.0, 0.0, 0.0] +| 2 | [1.0, 1.0, 1.0, 0.0, 0.0, 0.0] | 3 | [1.0, 0.0, 1.0, 0.0, 0.0, 0.0] -| 4 | [1.0, 1.0, 0.0, 0.0, 0.0, 0.0] -| 5 | [1.0, 1.0, 1.0, 0.0, 0.0, 0.0] +| 4 | [1.0, 1.0, 1.0, 0.0, 0.0, 0.0] +| 5 | [1.0, 1.0, 0.0, 0.0, 0.0, 0.0] | 6 | [1.0, 0.0, 0.0, 0.0, 0.0, 1.0] | 7 | [1.0, 0.0, 1.0, 0.0, 0.0, 0.0] | 8 | [1.0, 0.0, 0.0, 0.0, 0.0, 1.0] | 9 | [1.0, 0.0, 0.0, 0.0, 0.0, 1.0] -| 10 | [1.0, 0.0, 0.0, 0.0, 1.0, 0.0] +| 10 | [1.0, 0.0, 1.0, 0.0, 0.0, 0.0] |=== -- @@ -611,17 +634,17 @@ YIELD nodeId, embedding .Results |=== | nodeId | embedding -| 0 | [0.0, 0.8660253882408142, -0.8660253882408142, 0.0] -| 1 | [0.0, 0.8660253882408142, -0.8660253882408142, 0.0] -| 2 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] +| 0 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] +| 1 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] +| 2 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] | 3 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] -| 4 | [0.0, 0.8660253882408142, -0.8660253882408142, 0.0] -| 5 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] +| 4 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] +| 5 | [0.0, 0.8660253882408142, -0.8660253882408142, 0.0] | 6 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] | 7 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] | 8 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] | 9 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] -| 10 | [0.0, 0.0, -0.8660253882408142, 0.0] +| 10 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] |=== -- diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/index.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/index.adoc index 9cb9a082d4d..b672a3ebb0c 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/index.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/index.adoc @@ -13,8 +13,6 @@ The Neo4j Graph Data Science library contains the following node embedding algor * Beta ** xref:machine-learning/node-embeddings/graph-sage.adoc[GraphSAGE] ** xref:machine-learning/node-embeddings/node2vec.adoc[Node2Vec] - -* Alpha ** xref:machine-learning/node-embeddings/hashgnn.adoc[HashGNN] diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc index 50309dbd7e0..84af4a99b14 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc @@ -7,6 +7,13 @@ include::partial$/operations-reference/beta-note.adoc[] +[TIP] +==== +Node classification pipelines are featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/ml-pipelines-node-classification.ipynb[Machine learning pipelines: Node classification] +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning] +==== Node Classification is a common machine learning task applied to graphs: training models to classify nodes. Concretely, Node Classification models are used to predict the classes of unlabeled nodes as a node properties based on other node properties. diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc index 1d75fa50222..ecb96a378e0 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc @@ -6,6 +6,12 @@ include::partial$/operations-reference/alpha-note.adoc[] +[TIP] +==== +Node regression pipelines are featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression with Subgraph and Graph Sample projections] +==== Node Regression is a common machine learning task applied to graphs: training models to predict node property values. Concretely, Node Regression models are used to predict the value of node property based on other node properties. diff --git a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc index 263a7fcc9f4..56afdf9793d 100644 --- a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc +++ b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc @@ -44,8 +44,9 @@ YIELD .Configuration [opts="header",cols="1,1,1,4"] |=== -| Name | Type | Default | Description -| concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. +| Name | Type | Default | Description +| concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. +| includeGraphs | Boolean | true | Flag to decide whether only models or also graphs should be backed up. |=== .Results diff --git a/doc/modules/ROOT/pages/management-ops/create-cypher-db.adoc b/doc/modules/ROOT/pages/management-ops/create-cypher-db.adoc index 309948fef95..27decb76254 100644 --- a/doc/modules/ROOT/pages/management-ops/create-cypher-db.adoc +++ b/doc/modules/ROOT/pages/management-ops/create-cypher-db.adoc @@ -1,4 +1,4 @@ -[[create-cypher-db]] +[[cypher-on-gds]] = Cypher on GDS graph :description: This chapter explains how to execute Cypher queries on named graphs in the Neo4j Graph Data Science library. @@ -23,14 +23,13 @@ That database will then use data from the projected graph as compared to the sto Although it is possible to execute arbitrary Cypher queries on the database created by the `gds.alpha.create.cypherdb` procedure, not every aspect of Cypher is implemented yet. Some known limitations are listed below: -* Dropping the newly created database -** Restarting the DBMS will remove the database instead -* Writes -** All queries that attempt to write things, such as nodes, properties or labels, will fail +* Some writes will fail +** Creating new nodes and adding node labels +** Everything related to relationships [[create-cypher-db-syntax]] -== Syntax +== Create database syntax [.create-cypher-db-syntax] -- @@ -157,3 +156,28 @@ MATCH (n:Person)-[:KNOWS]->(m:Person) RETURN n.age AS age1, m.age AS age2 |=== We can see that the returned ages correspond to the structure of the original graph. + + +[[drop-cypher-db]] +== Dropping a GDS database + +As described above, in-memory GDS databases are impermanent and will be removed when the DBMS is shut down. +If we need to drop the GDS database earlier, there are 2 ways to achieve this: + 1. Using an administrative cypher command against the system database (`DROP DATABASE `) + 2. Using the <> procedure + +[[drop-cypher-db-procedure-syntax]] +=== Drop database syntax + +[.drop-cypher-db-syntax] +-- +[source, cypher, role=noplay] +---- +CALL gds.alpha.drop.cypherdb( + dbName: String +) +YIELD + dbName: String, + dropMillis: Integer +---- +-- diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index a9c6f08c84d..ba37df270e8 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -41,6 +41,16 @@ This chapter explains the available graph catalog operations. | xref:graph-exists.adoc[gds.graph.exists] | Checks if a named graph is stored in the catalog. |=== +== Modifying the graph catalog + +.Graph catalog update operations: +[opts=header,cols="1m,1"] +|=== +| Name | Description +| xref:graph-catalog-node-ops.adoc[gds.alpha.graph.nodeLabel.mutate] | Computes and adds a new node label to the graph. +| xref:graph-catalog-relationship-ops.adoc[gds.beta.graph.relationships.toUndirected] | Converts relationship of a given type in a graph from directed to undirected. +|=== + .Graph catalog export operations: [opts=header,cols="1m,1"] diff --git a/doc/modules/ROOT/pages/management-ops/index.adoc b/doc/modules/ROOT/pages/management-ops/index.adoc index b2645bdcf69..bb2c4a3bb63 100644 --- a/doc/modules/ROOT/pages/management-ops/index.adoc +++ b/doc/modules/ROOT/pages/management-ops/index.adoc @@ -13,4 +13,3 @@ This chapter is divided into the following sections: * xref:management-ops/create-cypher-db.adoc[Cypher on GDS graph] * xref:management-ops/administration.adoc[Administration] * xref:management-ops/backup-restore.adoc[Backup and Restore] -* xref:management-ops/defaults-and-limits.adoc[Defaults and Limits] diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc index c484061a1c8..8c1b7d2d7c3 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc @@ -26,13 +26,16 @@ The graph generation is parameterized by three dimensions: [[graph-generation-syntax]] == Syntax - -.The following describes the API for running the algorithm +[.include-with-graph-generate] +==== +.The following describes the API for running the graph generation procedure [source, cypher, role=noplay] ---- -CALL gds.beta.graph.generate(graphName: String, nodeCount: Integer, averageDegree: Integer, { - relationshipDistribution: String, - relationshipProperty: Map +CALL gds.beta.graph.generate( + graphName: String, + nodeCount: Integer, + averageDegree: Integer, + configuration: Map }) YIELD name, nodes, relationships, generateMillis, relationshipSeed, averageDegree, relationshipDistribution, relationshipProperty ---- @@ -72,6 +75,7 @@ YIELD name, nodes, relationships, generateMillis, relationshipSeed, averageDegre | relationshipDistribution | String | The probability distribution method used to connect generated nodes. | relationshipProperty | String | The configuration of the generated relationship property. |=== +==== [[graph-generation-distribution]] == Relationship Distribution @@ -112,3 +116,115 @@ Currently, there are two supported methods to generate relationship properties: * `FIXED` - Assigns a fixed value to every relationship. The `value` parameter must be set. * `RANDOM` - Assigns a random value between the lower (`min`) and upper (`max`) bound. + +[[graph-generation-example]] +== Examples + +In the following we will demonstrate the usage of the random graph generation procedure. + +[[graph-generation-unweighted]] +=== Generating unweighted graphs + +[role=query-example,group=unweighted] +-- +.The following will produce a graph with unweighted relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.generate('graph',5,2, {relationshipSeed:19}) +YIELD name, nodes, relationships, relationshipDistribution +---- + +.Results +[opts="header"] +|=== +| name | nodes | relationships | relationshipDistribution +| "graph"| 5 | 10 | "UNIFORM" +|=== +-- + +A new in-memory graph called `graph` with `5` nodes and `10` relationships has been created and added to the graph catalog. +We can examine its topology with the `gds.beta.graph.relationships` procedure. + +[role=query-example,group=unweighted] +-- +.The following will show the produced relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.relationships.stream('graph') +YIELD sourceNodeId,targetNodeId +RETURN sourceNodeId as source, targetNodeId as target +ORDER BY source ASC,target ASC +---- + +.Results +[opts="header"] +|=== +| source |target +| 0 | 1 +| 0 | 2 +| 1 | 0 +| 1 | 4 +| 2 | 1 +| 2 | 4 +| 3 | 0 +| 3 | 1 +| 4 | 0 +| 4 | 3 +|=== +-- + +[[graph-generation-weighted]] +=== Generating weighted graphs + +To generated graphs with weighted relationships we must specify the `relationshipProperty` parameter as discussed xref:graph-generation-relationship-property[above]. + +[role=query-example,group=weighted] +-- +.The following will produce a graph with weighted relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.generate('weightedGraph',5,2, {relationshipSeed:19, + relationshipProperty: {type: 'RANDOM', min: 5.0, max: 10.0, name: 'score'}}) +YIELD name, nodes, relationships, relationshipDistribution +---- + +.Results +[opts="header"] +|=== +|name| nodes | relationships | relationshipDistribution +| "weightedGraph"| 5 | 10 | "UNIFORM" +|=== +-- + +The produced graph, `weightedGraph`, has a property named `score` containing a random value between 5.0 and 10.0 for each relationship. +We can use `gds.graph.relationshipProperty.stream` to stream the relationships of the graph along with their score values. + +[role=query-example,group=weighted] +-- +.The following will show the produced relationships +[source,cypher,role=noplay] +---- +CALL gds.graph.relationshipProperty.stream('weightedGraph','score') +YIELD sourceNodeId, targetNodeId, propertyValue +RETURN sourceNodeId as source, targetNodeId as target, propertyValue as score +ORDER BY source ASC,target ASC, score +---- + +.Results +[opts="header"] +|=== +| source |target | score +| 0 | 1 | 6.258381821615686 +| 0 | 2 | 6.791408433596591 +| 1 | 4 | 8.747179900968224 +| 1 | 4 | 9.469695236791349 +| 2 | 0 | 7.061710127800056 +| 2 | 1 | 5.060444167785128 +| 3 | 1 | 6.308266834622538 +| 3 | 4 | 9.040323743901354 +| 4 | 1 | 7.939688205556302 +| 4 | 1 | 7.988277646384441 +|=== +-- + +Notice that despite `graph` and `weightedGraph` having the same `relationshipSeed`, their actual topology differs. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc index dbfffe41287..a4c6412c1d9 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc @@ -3,22 +3,17 @@ :description: This section details projecting GDS graphs using `Cypher` aggregations. +Using Cypher aggregations is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. +Cypher aggregations are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). -A projected graph can be stored in the catalog under a user-defined name. -Using that name, the graph can be referred to by any algorithm in the library. -This allows multiple algorithms to use the same graph without having to project it on each algorithm run. -Using Cypher aggregations is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. -Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). +== Considerations -[NOTE] --- -There is also a way to generate a random graph, see xref:management-ops/projections/graph-generation.adoc[Graph Generation] documentation for more details. --- +=== Lifecycle [NOTE] -- -The projected graph will reside in the catalog until: +The projected graphs will reside in the catalog until either: - the graph is dropped using xref:graph-drop.adoc[gds.graph.drop] - the Neo4j database from which the graph was projected is stopped or dropped @@ -26,12 +21,26 @@ The projected graph will reside in the catalog until: -- +=== Node property support + +Cypher aggregations can only project a limited set of node property types from a Cypher query. +The xref:management-ops/node-properties.adoc#node-properties-supported[Node Properties page] details which node property types are supported. +Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Cypher aggregation. + +=== Selection of node properties and labels + +If a node occurs multiple times, the node properties and labels of the first occurrence will be used for the projection. +This is important when a node can be a source node as well as a target node and their configuration differs. +Relevant configuration options are `sourceNodeProperties`, `targetNodeProperties`, `sourceNodeLabels` and `targetNodeLabels`. + + [[graph-project-cypher-aggregation-syntax]] == Syntax A Cypher aggregation is used in a query as an aggregation over the relationships that are being projected. It takes three mandatory arguments: `graphName`, `sourceNode` and `targetNode`. -In addition, the optional `sourceNodeProperties`, `targetNodeProperties`, and `relationshipProperties` parameters allows us to project properties. +In addition, two optional parameters can be used to project node properties and labels (`nodesConfig`) or relationship properties and type (`relationshipConfig`). +The optional `configuration` parameter can be used for general configuration of the projection such as `readConcurrency`. [.graph-project-cypher-aggregation-syntax] -- @@ -53,15 +62,35 @@ RETURN gds.alpha.graph.project( ---- .Parameters -[opts="header",cols="1,1,8"] +[opts="header",cols="2,1,7"] |=== | Name | Optional | Description | graphName | no | The name under which the graph is stored in the catalog. | sourceNode | no | The source node of the relationship. Must not be null. | targetNode | yes | The target node of the relationship. The targetNode can be null (for example due to an `OPTIONAL MATCH`), in which case the source node is projected as an unconnected node. -| nodesConfig | yes | Properties and Labels configuration for the source and target nodes. -| relationshipConfig | yes | Properties and Type configuration for the relationship. -| configuration | yes | Additional parameters to configure the cypher aggregation projection. +| <> | yes | Properties and labels configuration for the source and target nodes. +| <> | yes | Properties and type configuration for the relationship. +| <> | yes | Additional parameters to configure the projection. +|=== + +[[graph-project-cypher-aggregation-syntax-nodesConfig]] +.Nodes configuration +[opts="header",cols="1,1,1,4"] +|=== +| Name | Type | Default | Description +| sourceNodeProperties | Map | {} | The properties of the source node. +| targetNodeProperties | Map | {} | The properties of the target node. +| sourceNodeLabels | List of String or String | [] | The label(s) of the source node. +| targetNodeLabels | List of String or String | [] | The label(s) of the source node. +|=== + +[[graph-project-cypher-aggregation-syntax-relationshipConfig]] +.Relationship configuration +[opts="header",cols="1,1,1,4"] +|=== +| Name | Type | Default | Description +| properties | Map | {} | The properties of the source node. +| relationshipType | String | '*' | The type of the relationship. |=== [[graph-project-cypher-aggregation-syntax-configuration]] @@ -228,7 +257,7 @@ WITH gds.alpha.graph.project( relationshipType: type(r) } ) AS g -RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels ---- .Results @@ -265,13 +294,65 @@ The value for `relationshipType` must be a `String`: === Relationship orientation The native projection supports specifying an orientation per relationship type. -The Cypher Aggregation will treat every relationship returned by the relationship query as if it was in `NATURAL` orientation. -It is thus not possible to project graphs in `UNDIRECTED` or `REVERSE` orientation when Cypher projections are used. +The Cypher Aggregation will treat every relationship returned by the relationship query as if it was in `NATURAL` orientation by default. -[NOTE] +==== Reverse relationships + +The orientation of a relationship can be reversed by switching the source and target nodes. + +[role=query-example] -- -Some algorithms require that the graph was loaded with `UNDIRECTED` orientation. -These algorithms can not be used with a graph projected by a Cypher Aggregation. +.Project `Person` and `Book` nodes and `KNOWS` and `READ` relationships: +[source, cypher, role=noplay] +---- +MATCH (source)-[r:KNOWS|READ]->(target) +WHERE source:Book OR source:Person +WITH gds.alpha.graph.project( + 'graphWithReverseRelationships', + target, + source, + {}, + {} +) as g +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels +---- + +.Results +[opts="header", cols="1,1,1"] +|=== +| graph | nodes | rels +| "graphWithReverseRelationships" | 5 | 6 +|=== +-- + +==== Undirected relationships + +Relationships can be projected as undirected by specifying the `undirectedRelationshipTypes` parameter. + +[role=query-example] +-- +.Project `Person` and `Book` nodes and `KNOWS` and `READ` relationships: +[source, cypher, role=noplay] +---- +MATCH (source)-[r:KNOWS|READ]->(target) +WHERE source:Book OR source:Person +WITH gds.alpha.graph.project( + 'graphWithUndirectedRelationships', + source, + target, + {}, + {}, + {undirectedRelationshipTypes: ['*']} +) as g +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels +---- + +.Results +[opts="header", cols="1,1,1"] +|=== +| graph | nodes | rels +| "graphWithUndirectedRelationships" | 5 | 12 +|=== -- @@ -300,7 +381,7 @@ WITH gds.alpha.graph.project( targetNodeProperties: target { age: coalesce(target.age, 18), price: coalesce(target.price, 5.0), .ratings } } ) as g -RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels ---- .Results diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-subgraph.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-subgraph.adoc index a2c6514c01c..a96b9a7bea6 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-subgraph.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-subgraph.adoc @@ -3,9 +3,15 @@ = Projecting a subgraph :description: This section details how to project subgraphs from existing graphs stored in the graph catalog of the Neo4j Graph Data Science library. - include::partial$/operations-reference/beta-note.adoc[] +[TIP] +==== +Subgraph projection is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression with Subgraph and Graph Sample projections] +==== + In GDS, algorithms can be executed on a named graph that has been filtered based on its xref:common-usage/running-algos.adoc#common-configuration-node-labels[node labels] and xref:common-usage/running-algos.adoc#common-configuration-relationship-types[relationship types]. However, that filtered graph only exists during the execution of the algorithm, and it is not possible to filter on property values. If a filtered graph needs to be used multiple times, one can use the subgraph catalog procedure to project a new graph in the graph catalog. diff --git a/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc b/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc index 97dd211363f..fdd93828c4d 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc @@ -12,6 +12,14 @@ include::partial$/operations-reference/alpha-note.adoc[] :weighted: include::partial$/algorithms/shared/algorithm-traits.adoc[] +{nbsp} + +[TIP] +==== +Random walk with restarts sampling is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/import-sample-export-gnn.ipynb[Sampling, Export and Integration with PyG example] +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression with Subgraph and Graph Sample projections] +==== == Introduction @@ -96,6 +104,7 @@ include::partial$/algorithms/common-configuration/common-stream-stats-configurat | restartProbability | Float | 0.1 | yes | The probability that a sampling random walk restarts from one of the start nodes. | startNodes | List of Integer | A node chosen uniformly at random | yes | IDs of the initial set of nodes of the original graph from which the sampling random walks will start. | nodeLabelStratification | Boolean | false | yes | If true, preserves the node label distribution of the original graph. +| randomSeed | Integer | n/a | yes | A random seed which is used for all randomness in the computation. Requires `concurrency = 1`. |=== .Results diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 2dde5be0fc4..d80ad724319 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.0-alpha05" +| "2.3.9" |=== -- diff --git a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc index 55ec88482d4..5753bcc6277 100644 --- a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc @@ -22,11 +22,12 @@ | xref:alpha-algorithms/one-hot-encoding.adoc[One Hot Encoding] | `_gds.alpha.ml.oneHotEncoding_` | xref:common-usage/debug-sysinfo.adoc[Status of the system] | `gds.debug.sysInfo` | xref:management-ops/create-cypher-db.adoc[Create an impermanent database backed by a projected graph] | `gds.alpha.create.cypherdb` +| xref:management-ops/create-cypher-db.adoc#drop-cypher-db[Drop an impermanent database backed by a projected graph] | `gds.alpha.drop.cypherdb` | xref:common-usage/monitoring-system.adoc[Get an overview of the system's workload and available resources] | `gds.alpha.systemMonitor` -| Back-up graphs and models to disk | `gds.alpha.backup` -| Restore persisted graphs and models to memory | `gds.alpha.restore` -| xref:management-ops/defaults-and-limits.adoc[List configured defaults] | `gds.alpha.config.defaults.list` -| xref:management-ops/defaults-and-limits.adoc[Configure a default] | `gds.alpha.config.defaults.set` -| xref:management-ops/defaults-and-limits.adoc#_limits_on_configuration_values[List configured limits] | `gds.alpha.config.limits.list` -| xref:management-ops/defaults-and-limits.adoc#_limits_on_configuration_values[Configure a limit] | `gds.alpha.config.limits.set` +| xref:management-ops/backup-restore.adoc[Back-up graphs and models to disk] | `gds.alpha.backup` +| xref:management-ops/backup-restore.adoc[Restore persisted graphs and models to memory] | `gds.alpha.restore` +| xref:production-deployment/defaults-and-limits.adoc[List configured defaults] | `gds.alpha.config.defaults.list` +| xref:production-deployment/defaults-and-limits.adoc[Configure a default] | `gds.alpha.config.defaults.set` +| xref:production-deployment/defaults-and-limits.adoc#_limits_on_configuration_values[List configured limits] | `gds.alpha.config.limits.list` +| xref:production-deployment/defaults-and-limits.adoc#_limits_on_configuration_values[Configure a limit] | `gds.alpha.config.limits.set` |=== diff --git a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc index aefb224da8f..74d6c20a128 100644 --- a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc @@ -239,7 +239,7 @@ | `gds.beta.node2vec.stream.estimate` | `gds.beta.node2vec.write` | `gds.beta.node2vec.write.estimate` -.8+<.^| xref:algorithms/influence-maximization/celf.adoc[Influence Maximization - CELF] +.8+<.^| xref:algorithms/celf.adoc[Influence Maximization - CELF] | `gds.beta.influenceMaximization.celf.mutate` | `gds.beta.influenceMaximization.celf.mutate.estimate` | `gds.beta.influenceMaximization.celf.stats` diff --git a/doc/modules/ROOT/pages/operations-reference/appendix-a.adoc b/doc/modules/ROOT/pages/operations-reference/appendix-a.adoc index 6a3556b5c5d..184abf3dede 100644 --- a/doc/modules/ROOT/pages/operations-reference/appendix-a.adoc +++ b/doc/modules/ROOT/pages/operations-reference/appendix-a.adoc @@ -11,3 +11,4 @@ This chapter contains a full listing of all operations in the Neo4j Graph Data S * xref:operations-reference/algorithm-references.adoc[Graph Algorithms] * xref:operations-reference/machine-learning-references.adoc[Machine Learning] * xref:operations-reference/additional-operation-references.adoc[Additional Operations] +* xref:operations-reference/configuration-settings.adoc[Configuration Settings] diff --git a/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc new file mode 100644 index 00000000000..be7a7b63262 --- /dev/null +++ b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc @@ -0,0 +1,216 @@ +[[configuration-settings1]] += Configuration Settings +:description: This section describes the available configuration settings in the Neo4j Graph Data Science library. + +This page describes the available configuration settings in GDS. +Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4j-conf/#neo4j-conf[neo4j.conf] file for details on how to use configuration settings. + +[.all-settings] +.All settings +[cols="2,2,1", caption =] +|=== +<.^| <> +| The maximum time to wait for the next command before aborting the import process. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Address that clients should use to connect to the GDS Arrow Flight Server. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| The batch size used for arrow property export. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Enable the GDS Arrow Flight Server. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Never activate server-side encryption for the GDS Arrow Flight Server. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Address the GDS Arrow Flight Server should bind to. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Sets the location of the file that contains the Neo4j Graph Data Science library license key. +| + +<.^| <> +| Sets the export location for file based exports. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Sets the location where persisted models are stored. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Enable progress logging tracking. +| + +<.^| <> +| Use maximum memory estimation in procedure memory guard. +| +|=== + +[[gds.arrow.abortion_timeout]] +[.setting-details] +.gds.arrow.abortion_timeout label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | The maximum time to wait for the next command before aborting the import process. +| Default Value | `10m` +| Valid Values | A duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`). +| Dynamic | `false` +|=== + +[[gds.arrow.advertised_listen_address]] +[.setting-details] +.gds.arrow.advertised_listen_address label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Address that clients should use to connect to the GDS Arrow Flight Server. +| Default Value | `:8491` +| Valid Values | A socket address in the format `hostname:port`, `hostname` or `:port`. If missing port or hostname it is acquired from `gds.arrow.listen_address`. +| Dynamic | `false` +|=== + +[[gds.arrow.batch_size]] +[.setting-details] +.gds.arrow.batch_size label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | The batch size used for arrow property export. +| Default Value | `10000` +| Valid Values | An integer. +| Dynamic | `true` +|=== + + +[[gds.arrow.enabled]] +[.setting-details] +.gds.arrow.enabled label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Enable the GDS Arrow Flight Server. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + + +[[gds.arrow.encryption.never]] +[.setting-details] +.gds.arrow.encryption.never label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Never activate server-side encryption for the GDS Arrow Flight Server. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + + +[[gds.arrow.listen_address]] +[.setting-details] +.gds.arrow.listen_address label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Address the GDS Arrow Flight Server should bind to. +| Default Value | `localhost:8491` +| Valid Values | A socket address in the format `hostname:port`, `hostname` or `:port`. +| Dynamic | `false` +|=== + + +[[gds.cluster.tx.max.size]] +[.setting-details] +.gds.cluster.tx.max.size label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +| Default Value | `100000` +| Valid Values | An integer, must be set greater than or equal to the value of `gds.cluster.tx.min.size`. +| Dynamic | `false` +|=== + + +[[gds.cluster.tx.min.size]] +[.setting-details] +.gds.cluster.tx.min.size label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +| Default Value | `10000` +| Valid Values | An integer. +| Dynamic | `false` +|=== + + +[[gds.enterprise.license_file]] +[.setting-details] +.gds.enterprise.license_file +[cols="1,4", caption =] +|=== +| Description | Sets the location of the file that contains the Neo4j Graph Data Science library license key. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== + + +[[gds.export.location]] +[.setting-details] +.gds.export.location +[cols="1,4", caption = ] +|=== +| Description | Sets the export location for file based exports. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== + + +[[gds.model.store_location]] +[.setting-details] +.gds.model.store_location label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Sets the location where persisted models are stored. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== + + +[[gds.progress_tracking_enabled]] +[.setting-details] +.gds.progress_tracking_enabled +[cols="1,4", caption =] +|=== +| Description | Enable progress logging tracking. +| Default Value | `true` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + + +[[gds.validate_using_max_memory_estimation]] +[.setting-details] +.gds.validate_using_max_memory_estimation +[cols="1,4", caption =] +|=== +| Description | Use maximum memory estimation in procedure memory guard. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== diff --git a/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc index c9465dbca24..6388133bb9d 100644 --- a/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc @@ -59,7 +59,7 @@ | `gds.beta.graph.export.csv` | `gds.beta.graph.export.csv.estimate` |xref:graph-catalog-relationship-ops.adoc#catalog-graph-stream-relationship-topology-example[Stream relationship topologies to the procedure caller] | `gds.beta.graph.relationships.stream` -.2+<.^|Convert directed relationships to undirected +.2+<.^|xref:graph-catalog-relationship-ops.adoc#catalog-graph-relationship-to-undirected-example[Convert directed relationships to undirected] | `gds.beta.graph.relationships.toUndirected` | `gds.beta.graph.relationships.toUndirected.estimate` @@ -75,7 +75,7 @@ |Description | Operation |Drop a graph property from a named graph | `gds.alpha.graph.graphProperty.drop` |Stream a graph property to the procedure caller | `gds.alpha.graph.graphProperty.stream` -|Sample a subgraph using random walk with restarts | `gds.alpha.graph.sample.rwr` -|Add node labels to the in-memory graph | `gds.alpha.graph.nodeLabel.mutate` -|Write node labels to the database | `gds.alpha.graph.nodeLabel.write` +|xref:management-ops/projections/rwr.adoc[Sample a subgraph using random walk with restarts] | `gds.alpha.graph.sample.rwr` +|xref:graph-catalog-node-ops.adoc#catalog-graph-mutate-node-label-example[Add node labels to the in-memory graph] | `gds.alpha.graph.nodeLabel.mutate` +|xref:graph-catalog-node-ops.adoc#catalog-graph-write-node-label-example[Write node labels to the database] | `gds.alpha.graph.nodeLabel.write` |=== diff --git a/doc/modules/ROOT/pages/production-deployment/fabric.adoc b/doc/modules/ROOT/pages/production-deployment/composite.adoc similarity index 53% rename from doc/modules/ROOT/pages/production-deployment/fabric.adoc rename to doc/modules/ROOT/pages/production-deployment/composite.adoc index 335dddc827b..91a31a605c5 100644 --- a/doc/modules/ROOT/pages/production-deployment/fabric.adoc +++ b/doc/modules/ROOT/pages/production-deployment/composite.adoc @@ -1,41 +1,62 @@ -[[fabric]] -= Using GDS and Fabric -:description: This section describes how the Neo4j Graph Data Science library can be used in a Neo4j Fabric deployment. +:page-aliases: production-deployment/fabric + +[[composite]] +// Putting "Fabric" in the header might help with searching for the +// case that users are not familiar with composite databases yet += Using GDS and composite databases (formerly known as Fabric) +:description: This section describes how the Neo4j Graph Data Science library can be used in a Neo4j composite database deployment. include::partial$/common-usage/not-on-aurads-note.adoc[] -Neo4j Fabric is a way to store and retrieve data in multiple databases, whether they are on the same Neo4j DBMS or in multiple DBMSs, using a single Cypher query. -For more information about Fabric itself, please visit the https://neo4j.com/docs/operations-manual/4.4/fabric/introduction/[Fabric documentation]. +Neo4j composite databases are a way to store and retrieve data in multiple databases, whether they are on the same Neo4j DBMS or in multiple DBMSs, using a single Cypher query. +For more information about Composite databases/Fabric itself, please visit the + +[.tabbed-example, caption = ] +==== + +[.include-with-neo4j-4.x] +===== +https://neo4j.com/docs/operations-manual/4.4/fabric/introduction/[Fabric documentation]. +===== + +[.include-with-neo4j-5.x] +===== +https://neo4j.com/docs/operations-manual/current/composite-databases/[Composite databases documentation]. +===== + +==== + +NOTE: For simplicity this documentation page further only mentions composite databases which are available from Neo4j 5.0 on. As GDS supports 4.x and 5.x Neo4j versions this documentation can be also applied to Fabric setups using the exact same queries and examples as shown below. -A typical Neo4j Fabric setup consists of two components: one or more shards that hold the data and one or more Fabric proxies that coordinate the distributed queries. -There are two ways of running the Neo4j Graph Data Science library in a Fabric deployment, both of which are covered in this section: +A typical Neo4j composite setup consists of two components: one or more shards (constituents) that hold the data and one composite database that coordinates the distributed queries. +There are two ways of running the Neo4j Graph Data Science library in a composite deployment, both of which are covered in this section: - . Running GDS on a Fabric <> - . Running GDS on a Fabric <> + . Running GDS on a Composite <> + . Running GDS on a Composite <> -[[fabric-shard]] +[[composite-shard]] == Running GDS on the Shards -In this mode of using GDS in a Fabric environment, the GDS operations are executed on the shards. -The graph projections and algorithms are then executed on each shard individually, and the results can be combined via the Fabric proxy. +In this mode of using GDS in a composite environment, the GDS operations are executed on the shards. +The graph projections and algorithms are then executed on each shard individually, and the results can be combined via the composite database. This scenario is useful, if the graph is partitioned into disjoint subgraphs across shards, i.e. there is no logical relationship between nodes on different shards. Another use case is to replicate the graph's topology across multiple shards, where some shards act as operational and others as analytical databases. -[[fabric-shard-setup]] +[[composite-shard-setup]] === Setup In this scenario we need to set up the shards to run the Neo4j Graph Data Science library. Every shard that will run the Graph Data Science library should be configured just as a standalone GDS database would be, for more information see xref:installation/index.adoc[Installation]. -The Fabric proxy nodes do not require any special configuration, i.e., the GDS library plugin does not need to be installed. -However, the proxy nodes should be configured to handle the amount of data received from the shards. +The composite database does not require any special configuration, i.e., the GDS library plugin does not need to be installed. +However, the Composite database should be configured to handle the amount of data received from the shards. -[[fabric-shard-examples]] +[[composite-shard-examples]] === Examples -Let's assume we have a Fabric setup with two shards. +Let's assume we have a composite setup with two shards. One shard functions as the operational database and holds a graph with the schema `(Person)-[KNOWS]->(Person)`. Every `Person` node also stores an identifying property `id` and the persons `name` and possibly other properties. @@ -46,7 +67,7 @@ First we need to project a named graph on the analytical database shard. [source, cypher, role=noplay] ---- CALL { - USE FABRIC_DB_NAME.ANALYTICS_DB + USE COMPOSITE_DB_NAME.ANALYTICS_DB CALL gds.graph.project('graph', 'Person', 'KNOWS') YIELD graphName RETURN graphName @@ -54,18 +75,18 @@ CALL { RETURN graphName ---- -Using Fabric, we can now calculate the PageRank score for each Person and join the results with the name of that Person. +Using the composite database, we can now calculate the PageRank score for each Person and join the results with the name of that Person. [source, cypher, role=noplay] ---- CALL { - USE FABRIC_DB_NAME.ANALYTICS_DB + USE COMPOSITE_DB_NAME.ANALYTICS_DB CALL gds.pagerank.stream('graph', {}) YIELD nodeId, score AS pageRank RETURN gds.util.asNode(nodeId).id AS personId, pageRank } CALL { - USE FABRIC_DB_NAME.OPERATIONAL_DB + USE COMPOSITE_DB_NAME.OPERATIONAL_DB WITH personId MATCH (p {id: personId}) RETURN p.name AS name @@ -78,32 +99,32 @@ The algorithm results are streamed to the proxy, together with the unique node i For every row returned by the first subquery, the operational database is then queried for the persons name, again using the unique node id to identify the `Person` node across the shards. -[[fabric-proxy]] -== Running GDS on the Fabric Proxy +[[composite-proxy]] +== Running GDS on the Composite database -In this mode of using GDS in a Fabric environment, the GDS operations are executed on the Fabric proxy server. +In this mode of using GDS in a composite environment, the GDS operations are executed on the Fabric proxy server. The graph projections are then using the data stored on the shards to construct the in-memory graph. -NOTE: Currently only xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] is supported for projecting in-memory graphs on a Fabric proxy. +NOTE: Currently only xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] is supported for projecting in-memory graphs on a Composite database. -Graph algorithms can then be executed on the Fabric proxy, similar to a single machine setup. -This scenario is useful, if a graph, that logically represents a single graph, is distributed to different Fabric shards. +Graph algorithms can then be executed on the composite database, similar to a single machine setup. +This scenario is useful, if a graph that logically represents a single graph is distributed to different Composite shards. -[[fabric-proxy-setup]] +[[composite-proxy-setup]] === Setup In this scenario we need to set up the proxy to run the Neo4j Graph Data Science library. -The dbms that manages the Fabric proxy database needs to have the GDS plugin installed and configured. +The dbms that manages the composite database needs to have the GDS plugin installed and configured. For more information see xref:installation/index.adoc[Installation]. The proxy node should also be configured to handle the amount of data received from the shards as well as executing graph projections and algorithms. Fabric shards do not need any special configuration, i.e., the GDS library plugin does not need to be installed. -[[fabric-proxy-examples]] +[[composite-proxy-examples]] === Examples -Let's assume we have a Fabric setup with two shards. +Let's assume we have a composite setup with two shards. Both shards function as the operational databases and hold graphs with the schema `(Person)-[KNOWS]->(Person)`. We now need to query the shards in order to drive the import process on the proxy node. @@ -111,11 +132,11 @@ We now need to query the shards in order to drive the import process on the prox [source, cypher, role=noplay] ---- CALL { - USE FABRIC_DB_NAME.FABRIC_SHARD_0_NAME + USE COMPOSITE_DB_NAME.COMPOSITE_SHARD_0_NAME MATCH (p:Person) OPTIONAL MATCH (p)-[:KNOWS]->(n:Person) RETURN p, n UNION - USE FABRIC_DB_NAME.FABRIC_SHARD_1_NAME + USE COMPOSITE_DB_NAME.COMPOSITE_SHARD_1_NAME MATCH (p:Person) OPTIONAL MATCH (p)-[:KNOWS]->(n:Person) RETURN p, n } diff --git a/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc new file mode 100644 index 00000000000..ff311af117e --- /dev/null +++ b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc @@ -0,0 +1,153 @@ +[[configuration-settings]] += GDS Configuration Settings +:description: This section describes the available configuration settings in the Neo4j Graph Data Science library. + + +This page describes the available configuration settings in GDS. +Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4j-conf/#neo4j-conf[neo4j.conf] file for details on how to use configuration settings. + +== GDS Enterprise Edition + +[.setting-details] +.gds.enterprise.license_file +[cols="1,4", caption =] +|=== +| Description | Sets the location of the file that contains the Neo4j Graph Data Science library license key. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== + + +[.enterprise-edition] +== GDS and Arrow + +[.setting-details] +.gds.arrow.abortion_timeout +[cols="1,4", caption =] +|=== +| Description | The maximum time to wait for the next command before aborting the import process. +| Default Value | `10m` +| Valid Values | A duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`). +| Dynamic | `false` +|=== + +[.setting-details] +.gds.arrow.advertised_listen_address +[cols="1,4", caption =] +|=== +| Description | Address that clients should use to connect to the GDS Arrow Flight Server. +| Default Value | `:8491` +| Valid Values | A socket address in the format `hostname:port`, `hostname` or `:port`. If missing port or hostname it is acquired from `gds.arrow.listen_address`. +| Dynamic | `false` +|=== + +[.setting-details] +.gds.arrow.batch_size +[cols="1,4", caption =] +|=== +| Description | The batch size used for arrow property export. +| Default Value | `10000` +| Valid Values | An integer. +| Dynamic | `true` +|=== + +[.setting-details] +.gds.arrow.enabled +[cols="1,4", caption =] +|=== +| Description | Enable the GDS Arrow Flight Server. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + +[.setting-details] +.gds.arrow.encryption.never +[cols="1,4", caption =] +|=== +| Description | Never activate server-side encryption for the GDS Arrow Flight Server. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + +[.setting-details] +.gds.arrow.listen_address +[cols="1,4", caption =] +|=== +| Description | Address the GDS Arrow Flight Server should bind to. +| Default Value | `localhost:8491` +| Valid Values | A socket address in the format `hostname:port`, `hostname` or `:port`. +| Dynamic | `false` +|=== + + +[.enterprise-edition] +== Neo4j Cluster + +[.setting-details] +.gds.cluster.tx.max.size +[cols="1,4", caption =] +|=== +| Description | Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +| Default Value | `100000` +| Valid Values | An integer, must be set greater than or equal to the value of `gds.cluster.tx.min.size`. +| Dynamic | `false` +|=== + +[.setting-details] +.gds.cluster.tx.min.size +[cols="1,4", caption =] +|=== +| Description | Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +| Default Value | `10000` +| Valid Values | An integer. +| Dynamic | `false` +|=== + + +== GDS Export + +[.setting-details] +.gds.export.location +[cols="1,4", caption = ] +|=== +| Description | Sets the export location for file based exports. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== + +[.setting-details] +.gds.model.store_location label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Sets the location where persisted models are stored. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== + + +== Miscellaneous + +[.setting-details] +.gds.progress_tracking_enabled +[cols="1,4", caption =] +|=== +| Description | Enable progress logging tracking. +| Default Value | `true` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + +[.setting-details] +.gds.validate_using_max_memory_estimation +[cols="1,4", caption =] +|=== +| Description | Use maximum memory estimation in procedure memory guard. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== diff --git a/doc/modules/ROOT/pages/management-ops/defaults-and-limits.adoc b/doc/modules/ROOT/pages/production-deployment/defaults-and-limits.adoc similarity index 100% rename from doc/modules/ROOT/pages/management-ops/defaults-and-limits.adoc rename to doc/modules/ROOT/pages/production-deployment/defaults-and-limits.adoc diff --git a/doc/modules/ROOT/pages/production-deployment/index.adoc b/doc/modules/ROOT/pages/production-deployment/index.adoc index 0b3b7911b6e..f818030f53e 100644 --- a/doc/modules/ROOT/pages/production-deployment/index.adoc +++ b/doc/modules/ROOT/pages/production-deployment/index.adoc @@ -5,7 +5,9 @@ This chapter is divided into the following sections: +* xref:production-deployment/defaults-and-limits.adoc[Defaults and Limits] * xref:production-deployment/transaction-handling.adoc[Transaction Handling] -* xref:production-deployment/fabric.adoc[Using GDS and Fabric] -* xref:production-deployment/causal-cluster.adoc[GDS with Neo4j Causal Cluster] +* xref:production-deployment/composite.adoc[Using GDS and Composite databases] +* xref:production-deployment/neo4j-cluster.adoc[GDS with Neo4j cluster] +* xref:production-deployment/configuration-settings.adoc[GDS Configuration Settings] * xref:production-deployment/feature-toggles.adoc[GDS Feature Toggles] diff --git a/doc/modules/ROOT/pages/production-deployment/causal-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc similarity index 70% rename from doc/modules/ROOT/pages/production-deployment/causal-cluster.adoc rename to doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 0e68a5870c8..3c4fd176d11 100644 --- a/doc/modules/ROOT/pages/production-deployment/causal-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -1,12 +1,14 @@ +:page-aliases: production-deployment/causal-cluster + [.enterprise-edition] [[cluster]] -= GDS with Neo4j Causal Cluster -:description: This section describes how the Neo4j Graph Data Science library can be used in a Neo4j Causal Cluster deployment. += GDS with Neo4j cluster +:description: This section describes how the Neo4j Graph Data Science library can be used in a Neo4j cluster deployment. include::partial$/common-usage/not-on-aurads-note.adoc[] -It is possible to run GDS as part of Neo4j Causal Cluster deployment. +It is possible to run GDS as part of Neo4j cluster deployment. Since GDS performs large computations with the full resources of the system it is not suitable to run on instances that serve the transactional workload of the cluster. @@ -15,65 +17,67 @@ Since GDS performs large computations with the full resources of the system it i [.tabbed-example, caption = ] ==== -[.include-with-neo4j-4x] -===== - -We make use of a _Read Replica_ instance to deploy the GDS library and process analytical workloads. -Calls to GDS `write` procedures are internally directed to the cluster `LEADER` instance via _server-side routing_. - -[NOTE] -====== -Please refer to the https://neo4j.com/docs/operations-manual/4.4/clustering/[official Neo4j documentation] for details on how to setup Neo4j Causal Cluster. -Note that the link points to the latest Neo4j 4.x version documentation and the configuration settings may differ from earlier versions. -====== - -* The cluster must contain at least one _Read Replica_ instance -** single _Core member_ and a _Read Replica_ is a valid scenario. -** GDS workloads are not load-balanced if there are more than one _Read Replica_ instances. -* Cluster should be configured to use https://neo4j.com/docs/operations-manual/4.4/clustering/internals/#causal-clustering-routing[server-side routing]. -* GDS plugin deployed on the _Read Replica_. -** A valid GDS Enterprise Edition license must be installed and configured on the _Read Replica_. -** The driver connection to operated GDS should be made using the `bolt://` protocol, or _server-policy routed_ to the _Read Replica_ instance. - -For more information on setting up, configuring and managing a Neo4j Causal Clustering, please refer to https://neo4j.com/docs/operations-manual/4.4/clustering/[the documentation]. -===== - [.include-with-neo4j-5x] ===== We make use of a _Secondary_ instance to deploy the GDS library and process analytical workloads. Calls to GDS `write` procedures are internally directed via _server-side routing_ to the cluster instance that is a `Writer` for the database we work on. -Neo4j 5.x supports different databases on the same machine to act as `Primary` or `Secondary` members of the cluster, we *do not* recommend mixed setup. -GDS should be installed on a machine that is not serving transactional load and does not participate in `Leader` elections. +Neo4j 5.x supports different databases on the same cluster instance to act as `Primary` or `Secondary` members of the cluster. +In order for GDS to function, all databases on the instance it is installed have to be `Secondary`, including the `system` database (see https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_server.cluster.system_database_mode[server.cluster.system_database_mode] and https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_initial.server.mode_constraint[initial.server.mode_constraint]). +GDS has compute-intensive OLAP workloads that may disrupt the cluster operations and we recommend GDS to be installed on an instance that is not serving transactional load and does not participate in `Leader` elections. [NOTE] ====== -Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/[official Neo4j documentation] for details on how to setup Neo4j Causal Cluster. +Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/setup/analytics-cluster/[official Neo4j documentation] for details on how to set up a Neo4j analytics cluster. Note that the link points to the latest Neo4j version documentation and the configuration settings may differ from earlier versions. ====== * The cluster must contain at least one _Secondary_ machine ** single _Primary_ and a _Secondary_ is a valid scenario. ** GDS workloads are not load-balanced if there are more than one _Secondary_ instances. -* Cluster should be configured to use https://neo4j.com/docs/operations-manual/current/clustering/internals/#causal-clustering-routing[server-side routing]. +* Cluster should be configured to use https://neo4j.com/docs/operations-manual/current/clustering/internals/#clustering-routing[server-side routing]. * GDS plugin deployed on the _Secondary_. ** A valid GDS Enterprise Edition license must be installed and configured on the _Secondary_. ** The driver connection to operated GDS should be made using the `bolt://` protocol to the _Secondary_ instance. -For more information on setting up, configuring and managing a Neo4j Causal Clustering, please refer to https://neo4j.com/docs/operations-manual/current/clustering/[the documentation]. +For more information on setting up, configuring and managing a Neo4j cluster, please refer to https://neo4j.com/docs/operations-manual/current/clustering/[the documentation]. [NOTE] ====== -When working with cluster configuration you should beware https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_dbms.config.strict_validation[strict config validation] in Neo4j. +When working with cluster configuration you should beware https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_server.config.strict_validation.enabled[strict config validation] in Neo4j. -When configuring GDS for a Read Replica you will introduce GDS-specific configuration into `neo4j.conf` - and that is fine because with the GDS plugin installed, Neo4j will happily validate those configuration items. +When configuring GDS for a Secondary instance you will introduce GDS-specific configuration into `neo4j.conf` - and that is fine because with the GDS plugin installed, Neo4j will happily validate those configuration items. However, you might not be able to reuse that same configuration file verbatim on the core cluster members, because there you will _not_ install GDS plugin, and thus Neo4j will _not_ be able to validate the GDS-specific configuration items. And validation failure would mean Neo4j would refuse to start. It is of course also possible to turn strict validation off. ====== ===== + +[.include-with-neo4j-4x] +===== + +We make use of a _Read Replica_ instance to deploy the GDS library and process analytical workloads. +Calls to GDS `write` procedures are internally directed to the cluster `LEADER` instance via _server-side routing_. + +[NOTE] +====== +Please refer to the https://neo4j.com/docs/operations-manual/4.4/clustering/[official Neo4j documentation] for details on how to setup Neo4j Causal Cluster. +Note that the link points to the latest Neo4j 4.x version documentation and the configuration settings may differ from earlier versions. +====== + +* The cluster must contain at least one _Read Replica_ instance +** single _Core member_ and a _Read Replica_ is a valid scenario. +** GDS workloads are not load-balanced if there are more than one _Read Replica_ instances. +* Cluster should be configured to use https://neo4j.com/docs/operations-manual/4.4/clustering/internals/#causal-clustering-routing[server-side routing]. +* GDS plugin deployed on the _Read Replica_. +** A valid GDS Enterprise Edition license must be installed and configured on the _Read Replica_. +** The driver connection to operated GDS should be made using the `bolt://` protocol, or _server-policy routed_ to the _Read Replica_ instance. + +For more information on setting up, configuring and managing a Neo4j Causal Clustering, please refer to https://neo4j.com/docs/operations-manual/4.4/clustering/[the documentation]. +===== + ==== == GDS Configuration diff --git a/doc/modules/ROOT/partials/algorithms/alpha/conductance/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/alpha/conductance/specific-configuration.adoc index 9746bf81cf4..0f7807cb18d 100644 --- a/doc/modules/ROOT/partials/algorithms/alpha/conductance/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/alpha/conductance/specific-configuration.adoc @@ -1 +1,2 @@ +| xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. | communityProperty | String | n/a | no | The node property that holds the community ID as an integer for each node. Note that only non-negative community IDs are considered valid and will have their conductance computed. diff --git a/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc index 231f3cc695c..958956007b8 100644 --- a/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc @@ -1,2 +1,2 @@ -| communityProperty | String | n/a | no | The node property that holds the community ID as an integer for each node. Note that only non-negative community IDs are considered valid and will have their conductance computed. -| relationshipWeightProperty | String | null | yes | Relationship Weight. +| xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. +| communityProperty | String | n/a | no | The node property that holds the community ID as an integer for each node. Note that only non-negative community IDs are considered valid and will have their modularity score computed. diff --git a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc index 3beba6c5e4c..ddfe72c918b 100644 --- a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc +++ b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc @@ -1,2 +1,6 @@ +ifeval::[{sequential} != true] | xref:common-usage/running-algos.adoc#common-configuration-concurrency[concurrency] | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. +endif::[] + | xref:common-usage/running-algos.adoc#common-configuration-jobid[jobId] | String | Generated internally | yes | An ID that can be provided to more easily track the algorithm's progress. +| xref:common-usage/running-algos.adoc#common-configuration-logProgress[logProgress] | Boolean | true | yes | If disabled the progress percentage will not be logged. diff --git a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration.adoc b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration.adoc index 55f9fbdac01..141226463eb 100644 --- a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration.adoc @@ -4,4 +4,5 @@ | Name | Type | Default | Optional | Description | xref:common-usage/running-algos.adoc#common-configuration-concurrency[concurrency] | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. Also provides the default value for 'readConcurrency' and 'writeConcurrency'. | xref:common-usage/running-algos.adoc#common-configuration-write-concurrency[writeConcurrency] | Integer | value of 'concurrency' | yes | The number of concurrent threads used for writing the result (applicable in WRITE mode). +| xref:common-usage/running-algos.adoc#common-configuration-logProgress[logProgress] | Boolean | true | yes | If disabled the progress percentage will not be logged. |=== diff --git a/doc/modules/ROOT/partials/algorithms/k-spanning-tree/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/k-spanning-tree/specific-configuration.adoc new file mode 100644 index 00000000000..4e5ad333b89 --- /dev/null +++ b/doc/modules/ROOT/partials/algorithms/k-spanning-tree/specific-configuration.adoc @@ -0,0 +1,4 @@ +| k | Number | n/a | no | The size of the tree to be returned +| sourceNode | Integer | null | n/a | The starting source node ID. +| xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. +| objective | String | 'minimum' | yes | If specified, the parameter dictates whether to seek a minimum or the maximum weight k-spanning tree. By default, the procedure looks for a minimum weight k-spanning tree. Permitted values are 'minimum' and 'maximum'. diff --git a/doc/modules/ROOT/partials/algorithms/kmeans/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/kmeans/specific-configuration.adoc index dc5d8c2a1f8..ac894342a41 100644 --- a/doc/modules/ROOT/partials/algorithms/kmeans/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/kmeans/specific-configuration.adoc @@ -1,4 +1,4 @@ -| nodeProperty | String | n/a | no | A node property to be used by the algorithm. +| nodeProperty | String | n/a | no | A node property corresponding to an array of floats used by K-Means to cluster nodes into communities. | k | Integer | 10 | yes | Number of desired clusters. | maxIterations | Integer | 10 | yes | The maximum number of iterations of K-Means to run. | deltaThreshold | Float | 0.05 | yes | Value as a percentage to determine when to stop early. If fewer than 'deltaThreshold * \|nodes\|' nodes change their cluster , the algorithm stops. Value must be between 0 (exclusive) and 1 (inclusive). diff --git a/doc/modules/ROOT/partials/algorithms/leiden/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/leiden/specific-configuration.adoc index 6bc900c22f6..ca18c820a4b 100644 --- a/doc/modules/ROOT/partials/algorithms/leiden/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/leiden/specific-configuration.adoc @@ -5,3 +5,4 @@ | xref:common-usage/running-algos.adoc#common-configuration-tolerance[tolerance] | Float | 0.0001 | yes | Minimum change in modularity between iterations. If the modularity changes less than the tolerance value, the result is considered stable and the algorithm returns. | includeIntermediateCommunities | Boolean | false | yes | Indicates whether to write intermediate communities. If set to false, only the final community is persisted. | xref:common-usage/running-algos.adoc#common-configuration-seed-property[seedProperty] | String | n/a | yes | Used to set the initial community for a node. The property value needs to be a non-negative number. +| consecutiveIds | Boolean | false | yes | Flag to decide whether component identifiers are mapped into a consecutive id space (requires additional memory). Cannot be used in combination with the `includeIntermediateCommunities` flag. diff --git a/doc/modules/ROOT/partials/algorithms/page-rank/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/page-rank/specific-configuration.adoc index a86222bb207..3c502a76456 100644 --- a/doc/modules/ROOT/partials/algorithms/page-rank/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/page-rank/specific-configuration.adoc @@ -2,5 +2,5 @@ | xref:common-usage/running-algos.adoc#common-configuration-max-iterations[maxIterations] | Integer | 20 | yes | The maximum number of iterations of Page Rank to run. | xref:common-usage/running-algos.adoc#common-configuration-tolerance[tolerance] | Float | 0.0000001 | yes | Minimum change in scores between iterations. If all scores change less than the tolerance value the result is considered stable and the algorithm returns. | xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. -| sourceNodes | List or Node or Number | [] | yes | The nodes or node ids to use for computing Personalized Page Rank. +| sourceNodes | List of Node or Number | [] | yes | The nodes or node ids to use for computing Personalized Page Rank. | scaler | String | None | yes | The name of the scaler applied for the final scores. Supported values are `None`, `MinMax`, `Max`, `Mean`, `Log`, `L1Norm`, `L2Norm` and `StdScore`. diff --git a/doc/modules/ROOT/partials/algorithms/wcc/specific-configuration-write.adoc b/doc/modules/ROOT/partials/algorithms/wcc/specific-configuration-write.adoc new file mode 100644 index 00000000000..63501a43b43 --- /dev/null +++ b/doc/modules/ROOT/partials/algorithms/wcc/specific-configuration-write.adoc @@ -0,0 +1,2 @@ +include::partial$/algorithms/wcc/specific-configuration.adoc[] +| minComponentSize | Integer | 0 | yes | Only nodes inside communities larger or equal the given value will be written to the underlying Neo4j database. diff --git a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc new file mode 100644 index 00000000000..2e9f427f58e --- /dev/null +++ b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc @@ -0,0 +1,28 @@ +[[introduction-editions]] += Editions + +The Neo4j Graph Data Science library is available in two editions. +By default, GDS will operate as the Community Edition. +To unlock Enterprise Edition features, a valid Neo4j Graph Data Science Enterprise license file is required. +See xref:installation/installation-enterprise-edition.adoc[] for how to configure the license. + +* The open source Community Edition: +** Includes all algorithms. +** Limits the catalog operations to manage graphs and models. + Unavailable operations are listed under the Enterprise Edition below. +** Limits the xref:installation/System-requirements.adoc#system-requirements-cpu[concurrency to maximum 4 CPU cores]. +** Limits the capacity of the model catalog to 3 models. + +* The Neo4j Graph Data Science library Enterprise Edition: +** Supports running on xref:installation/System-requirements.adoc#system-requirements-cpu[any amount of CPU cores]. +** Supports running GDS write workloads as part of a xref::production-deployment/neo4j-cluster.adoc[Neo4j cluster deployment]. +** Supports capacity and load xref::common-usage/monitoring-system.adoc[monitoring]. +** Supports extended graph catalog features, including: +*** Graph xref::management-ops/backup-restore.adoc[backup and restore]. +*** Data import and export via xref:installation/installation-apache-arrow.adoc[Apache Arrow]. +** Supports extended model catalog features, including: +*** Storing any number of models in the model catalog. +*** Sharing of models between users, through xref:model-catalog/publish.adoc[publishing]. +*** Model xref:model-catalog/store.adoc#model-catalog-store-ops[persistence to disk]. +** Supports an xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[optimized graph implementation], enabled by default. +** Supports the configuration of xref:production-deployment/defaults-and-limits.adoc[defaults and limits]. diff --git a/doc/package.json b/doc/package.json index 672e5ff5c5d..8ef75e66def 100644 --- a/doc/package.json +++ b/doc/package.json @@ -1,17 +1,14 @@ { "name": "graph-data-science", "version": "2.3.0-preview", - "description": "Neo4j Graph Data Science Library", + "description": "Neo4j Graph Data Science", "main": "server.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon -e adoc --exec \"npm run build && npm run serve\"", - "start:publish": "nodemon -e adoc --exec \"npm run build:publish && npm run serve\"", "serve": "node server.js", "build": "antora preview.yml --stacktrace --log-format=pretty", - "build:publish": "antora publish.yml --stacktrace --log-format=pretty", - "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json", - "publish-verify": "antora --stacktrace --fetch publish.yml --log-format=json --log-file ./build/log/log.json" + "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json" }, "keywords": [ "antora", diff --git a/doc/publish.yml b/doc/publish.yml deleted file mode 100644 index 09341aeb3b8..00000000000 --- a/doc/publish.yml +++ /dev/null @@ -1,60 +0,0 @@ -site: - title: Neo4j Graph Data Science - url: https://neo4j.com/docs - start_page: graph-data-science:ROOT:index.adoc - -content: - sources: - - url: https://github.com/neo-technology/graph-analytics.git - branches: ['2.2', 'master'] - start_path: public/doc - include: public/doc/ - exclude: - - '!**/_includes/*' - - '!**/readme.adoc' - - '!**/README.adoc' -ui: - bundle: - url: https://s3-eu-west-1.amazonaws.com/static-content.neo4j.com/build/ui-bundle-latest.zip - snapshot: true - output_dir: /assets - -urls: - html_extension_style: indexify - -antora: - extensions: - - require: "@neo4j-antora/antora-modify-sitemaps" - sitemap_version: '2.2' - sitemap_loc_version: current - move_sitemaps_to_components: 'true' - -asciidoc: - extensions: - - "@neo4j-documentation/remote-include" - - "@neo4j-documentation/macros" - - "@neo4j-antora/antora-page-roles" - - "@neo4j-antora/antora-table-footnotes" - attributes: - page-theme: docs - page-type: Docs - page-search-type: Docs - page-search-site: Reference Docs - page-canonical-root: /docs - page-pagination: true - page-no-canonical: true - page-origin-private: true - page-hide-toc: false - page-mixpanel: 4bfb2414ab973c741b6f067bf06d5575 - # page-cdn: /static/assets - includePDF: false - sectnums: true - sectnumlevel: 3 - doctype: book - nonhtmloutput: "" - # sectnums: true, removed so they are off by default - # sectnumlevel: 3, - experimental: '' - copyright: 2022 - common-license-page-uri: https://neo4j.com/docs/license/ - operations-manual-base-uri: https://neo4j.com/docs/operations-manual/ diff --git a/etc/spotbugs/spotbugs-exclude.xml b/etc/spotbugs/spotbugs-exclude.xml index 24d2964bf79..706ee305eed 100644 --- a/etc/spotbugs/spotbugs-exclude.xml +++ b/etc/spotbugs/spotbugs-exclude.xml @@ -697,7 +697,7 @@ - + diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index 2eccbd2a850..2a1996a0af3 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -2,12 +2,12 @@ plugins { // Apply the java plugin to add support for Java id 'java' // Used for building a standalone jar - id 'com.github.johnrengelman.shadow' version '4.0.4' + id 'com.github.johnrengelman.shadow' version '7.1.2' } ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.0-alpha05' + gdsVersion = '2.3.9' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java b/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java index 6261eae4252..4da733a0961 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java +++ b/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java @@ -42,8 +42,6 @@ import org.neo4j.logging.Log; import org.neo4j.logging.NullLog; -import static org.neo4j.gds.utils.StringFormatting.toLowerCaseWithLocale; - // TODO Remove the @Nullable annotations once the EstimationCli uses ProcedureExecutors @ValueClass public interface ExecutionContext { @@ -96,7 +94,7 @@ default DatabaseId databaseId() { @Value.Lazy default boolean containsOutputField(String fieldName) { return callContext().outputFields() - .anyMatch(field -> toLowerCaseWithLocale(field).equals(fieldName)); + .anyMatch(field -> field.equals(fieldName)); } @Value.Lazy diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 0d8e1206ea0..34cecb93311 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,9 +1,15 @@ ext { neos = [ - '4.4' : properties.getOrDefault('neo4jVersion44', '4.4.16'), - '5.1' : properties.getOrDefault('neo4jVersion51', '5.1.0'), - '5.2' : properties.getOrDefault('neo4jVersion52', '5.2.0'), - '5.3' : properties.getOrDefault('neo4jVersion53', '5.3.0'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.22'), + '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), + '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), + '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), + '5.4': properties.getOrDefault('neo4jVersion54', '5.4.0'), + '5.5': properties.getOrDefault('neo4jVersion55', '5.5.0'), + '5.6': properties.getOrDefault('neo4jVersion56', '5.6.0'), + '5.7': properties.getOrDefault('neo4jVersion57', '5.7.0'), + '5.8': properties.getOrDefault('neo4jVersion58', '5.8.0'), + '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), ] neo4jDefault = neos.'4.4' @@ -17,6 +23,24 @@ ext { '5.2': '2.13.8', '5.3': '2.13.8', '5.4': '2.13.8', + '5.5': '2.13.8', + '5.6': '2.13.8', + '5.7': '2.13.10', + '5.8': '2.13.10', + '5.9': '2.13.10', + ] + + log4js = [ + '4.4': '2.17.0', + '5.1': '2.18.0', + '5.2': '2.19.0', + '5.3': '2.19.0', + '5.4': '2.19.0', + '5.5': '2.19.0', + '5.6': '2.19.0', + '5.7': '2.20.0', + '5.8': '2.20.0', + '5.9': '2.20.0', ] ver = [ @@ -53,6 +77,7 @@ ext { 'jqwik' : '1.6.5', 'junit4' : '4.13.2', 'junit5bom' : '5.9.1', + 'log4j' : log4js[neo4j_minor], 'memoryfilesystem' : '2.2.0', 'mockito' : '4.11.0', 'mockito-junit-jupiter': '4.11.0', diff --git a/gradle/forbidden-apis.gradle b/gradle/forbidden-apis.gradle index 0020e170a22..b75dce44b81 100644 --- a/gradle/forbidden-apis.gradle +++ b/gradle/forbidden-apis.gradle @@ -8,6 +8,7 @@ if (shouldForbiddenApis) { proj.apply plugin: 'de.thetaphi.forbiddenapis' proj.forbiddenApis { signaturesFiles += files("$publicDir/etc/forbidden-apis") + ignoreSignaturesOfMissingClasses = true suppressAnnotations = ["org.neo4j.gds.annotation.SuppressForbidden"] bundledSignatures = ['jdk-system-out'] diff --git a/gradle/version.gradle b/gradle/version.gradle index b32a9106fb1..c9ebf7ac4ef 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,3 +1,6 @@ ext { - gdsVersion = '2.3.0-alpha05' + gdsBaseVersion = '2.3.9' + gdsAuraVersion = '23' + + gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/ElementProjection.java b/graph-projection-api/src/main/java/org/neo4j/gds/ElementProjection.java index eb2ca77f4e9..33cbc328406 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/ElementProjection.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/ElementProjection.java @@ -102,7 +102,7 @@ default Self addProperty( ) { inlineBuilder() .propertiesBuilder() - .addMapping(propertyKey, neoPropertyKey, defaultValue, aggregation); + .addMapping(PropertyMapping.of(propertyKey, neoPropertyKey, defaultValue, aggregation)); return (Self) this; } @@ -126,7 +126,7 @@ default void buildProperties() { static final class InlinePropertiesBuilder { private final Supplier getProperties; private final Consumer setProperties; - private AbstractPropertyMappings.Builder propertiesBuilder; + private PropertyMappings.Builder propertiesBuilder; InlinePropertiesBuilder( Supplier getProperties, @@ -151,9 +151,9 @@ private void build() { } } - private AbstractPropertyMappings.Builder propertiesBuilder() { + private PropertyMappings.Builder propertiesBuilder() { if (propertiesBuilder == null) { - propertiesBuilder = AbstractPropertyMappings.builder(); + propertiesBuilder = PropertyMappings.builder(); PropertyMappings properties = getProperties.get(); if (properties != null) { propertiesBuilder.from(properties); diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjection.java b/graph-projection-api/src/main/java/org/neo4j/gds/NodeProjection.java similarity index 87% rename from graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjection.java rename to graph-projection-api/src/main/java/org/neo4j/gds/NodeProjection.java index 0e2f2be99c7..a8803ef1f63 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjection.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/NodeProjection.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.jetbrains.annotations.Nullable; -import org.neo4j.gds.annotation.DataClass; +import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.core.ConfigKeyValidation; import org.neo4j.gds.utils.StringFormatting; @@ -29,10 +29,10 @@ import java.util.Map; import java.util.TreeMap; -@DataClass -public abstract class AbstractNodeProjection extends ElementProjection { +@ValueClass +public abstract class NodeProjection extends ElementProjection { - private static final NodeProjection ALL = of(PROJECT_ALL); + private static final NodeProjection ALL = fromString(PROJECT_ALL); public abstract String label(); @@ -50,7 +50,7 @@ public boolean projectAll() { public static final String LABEL_KEY = "label"; public static NodeProjection of(String label) { - return NodeProjection.of(label, PropertyMappings.of()); + return ImmutableNodeProjection.of(label, PropertyMappings.of()); } public static NodeProjection all() { @@ -85,7 +85,7 @@ public static NodeProjection fromString(@Nullable String label) { public static NodeProjection fromMap(Map map, NodeLabel nodeLabel) { validateConfigKeys(map); String label = String.valueOf(map.getOrDefault(LABEL_KEY, nodeLabel.name)); - return create(map, properties -> NodeProjection.of(label, properties)); + return create(map, properties -> ImmutableNodeProjection.of(label, properties)); } @Override @@ -102,9 +102,9 @@ void writeToObject(Map value) { public NodeProjection withAdditionalPropertyMappings(PropertyMappings mappings) { PropertyMappings newMappings = properties().mergeWith(mappings); if (newMappings == properties()) { - return (NodeProjection) this; + return this; } - return ((NodeProjection) this).withProperties(newMappings); + return ((ImmutableNodeProjection) this).withProperties(newMappings); } public static Builder builder() { @@ -116,7 +116,7 @@ private static void validateConfigKeys(Map map) { } @org.immutables.builder.Builder.AccessibleFields - public static final class Builder extends NodeProjection.Builder implements InlineProperties { + public static final class Builder extends ImmutableNodeProjection.Builder implements InlineProperties { private InlinePropertiesBuilder propertiesBuilder; diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java b/graph-projection-api/src/main/java/org/neo4j/gds/NodeProjections.java similarity index 74% rename from graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java rename to graph-projection-api/src/main/java/org/neo4j/gds/NodeProjections.java index 98cb3ea5a9c..be2fb2d0b2c 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/NodeProjections.java @@ -21,12 +21,13 @@ import org.immutables.value.Value; import org.jetbrains.annotations.Nullable; -import org.neo4j.gds.annotation.DataClass; +import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.utils.StringFormatting; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -36,9 +37,9 @@ import static org.neo4j.gds.NodeLabel.ALL_NODES; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -@DataClass +@ValueClass @Value.Immutable(singleton = true) -public abstract class AbstractNodeProjections extends AbstractProjections { +public abstract class NodeProjections extends AbstractProjections { public static final NodeProjections ALL = create(singletonMap(ALL_NODES, NodeProjection.all())); @@ -111,11 +112,11 @@ public static NodeProjections create(Map projections) "An empty node projection was given; at least one node label must be projected." ); } - return NodeProjections.of(unmodifiableMap(projections)); + return ImmutableNodeProjections.of(unmodifiableMap(projections)); } public static NodeProjections single(NodeLabel label, NodeProjection projection) { - return NodeProjections.of(Map.of(label, projection)); + return ImmutableNodeProjections.of(Map.of(label, projection)); } public static NodeProjections all() { @@ -124,7 +125,7 @@ public static NodeProjections all() { public NodeProjections addPropertyMappings(PropertyMappings mappings) { if (!mappings.hasMappings()) { - return NodeProjections.copyOf(this); + return ImmutableNodeProjections.copyOf(this); } Map newProjections = projections().entrySet().stream().collect(toMap( Map.Entry::getKey, @@ -149,7 +150,7 @@ public String labelProjection() { } public boolean isEmpty() { - return this == NodeProjections.of(); + return this == ImmutableNodeProjections.of(); } public Map toObject() { @@ -160,7 +161,7 @@ public Map toObject() { return value; } - public static Map toObject(AbstractNodeProjections nodeProjections) { + public static Map toObject(NodeProjections nodeProjections) { return nodeProjections.toObject(); } @@ -175,23 +176,39 @@ private static void validateIdentifierName(String identifier) { @Value.Check public void validatePropertyKeyMappings() { - var mapping = new HashMap(); + var seenMappings = new HashMap(); projections().values().stream() .flatMap(nodeProjection -> nodeProjection.properties().stream()) .forEach(propertyMapping -> { var propertyKey = propertyMapping.propertyKey(); - var neoKey = propertyMapping.neoPropertyKey(); - - if (mapping.containsKey(propertyKey) && !mapping.get(propertyKey).equals(neoKey)) { - throw new IllegalArgumentException(formatWithLocale( - "Specifying multiple neoPropertyKeys for the same property is not allowed, found propertyKey: %s, neoPropertyKeys: %s, %s.", - propertyKey, - neoKey, - mapping.get(propertyKey) - )); + + if (seenMappings.containsKey(propertyKey)) { + // we have another mapping with the same GDS key + var seenMapping = seenMappings.get(propertyKey); + + if (!Objects.equals(seenMapping.neoPropertyKey(), propertyMapping.neoPropertyKey())) { + throw new IllegalArgumentException(formatWithLocale( + "Specifying multiple neoPropertyKeys for the same property is not allowed, " + + "found propertyKey: `%s` with conflicting neoPropertyKeys: `%s`, `%s`.", + propertyKey, + propertyMapping.neoPropertyKey(), + seenMapping.neoPropertyKey() + )); + } + + if (!Objects.equals(seenMapping.defaultValue(), propertyMapping.defaultValue())) { + throw new IllegalArgumentException(formatWithLocale( + "Specifying different default values for the same property with identical neoPropertyKey is not allowed, " + + "found propertyKey: `%s` with conflicting default values: `%s`, `%s`.", + propertyKey, + propertyMapping.defaultValue().getObject(), + seenMapping.defaultValue().getObject() + )); + } } - mapping.put(propertyKey, neoKey); + + seenMappings.put(propertyKey, propertyMapping); }); } } diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractPropertyMappings.java b/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMappings.java similarity index 87% rename from graph-projection-api/src/main/java/org/neo4j/gds/AbstractPropertyMappings.java rename to graph-projection-api/src/main/java/org/neo4j/gds/PropertyMappings.java index 3f900bff305..9fbb7e3fd2a 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractPropertyMappings.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMappings.java @@ -19,12 +19,10 @@ */ package org.neo4j.gds; -import org.eclipse.collections.api.tuple.primitive.IntObjectPair; import org.immutables.builder.Builder.AccessibleFields; import org.immutables.value.Value; -import org.neo4j.gds.annotation.DataClass; +import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.utils.CollectionUtil; import java.util.Arrays; import java.util.Iterator; @@ -40,17 +38,17 @@ import static java.util.Collections.singletonMap; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -@DataClass +@ValueClass @Value.Immutable(singleton = true) -public abstract class AbstractPropertyMappings implements Iterable { +public abstract class PropertyMappings implements Iterable { public abstract List mappings(); public static PropertyMappings of(PropertyMapping... mappings) { if (mappings == null) { - return PropertyMappings.of(); + return ImmutablePropertyMappings.of(); } - return PropertyMappings.of(Arrays.asList(mappings)); + return ImmutablePropertyMappings.of(Arrays.asList(mappings)); } public static PropertyMappings fromObject(Object relPropertyMapping) { @@ -58,9 +56,9 @@ public static PropertyMappings fromObject(Object relPropertyMapping) { } public static PropertyMappings fromObject(Object relPropertyMapping, Aggregation defaultAggregation) { - if (relPropertyMapping instanceof PropertyMappings) { - PropertyMappings properties = (PropertyMappings) relPropertyMapping; - return PropertyMappings.builder().from(properties).withDefaultAggregation(defaultAggregation).build(); + if (relPropertyMapping instanceof ImmutablePropertyMappings) { + ImmutablePropertyMappings properties = (ImmutablePropertyMappings) relPropertyMapping; + return ImmutablePropertyMappings.builder().from(properties).withDefaultAggregation(defaultAggregation).build(); } if (relPropertyMapping instanceof String) { String propertyMapping = (String) relPropertyMapping; @@ -95,7 +93,7 @@ public static PropertyMappings fromObject(Object relPropertyMapping, Aggregation } } - public static Map toObject(AbstractPropertyMappings propertyMappings) { + public static Map toObject(PropertyMappings propertyMappings) { return propertyMappings.toObject(true); } @@ -112,10 +110,6 @@ public Iterator iterator() { return mappings().iterator(); } - public Stream> enumerate() { - return CollectionUtil.enumerate(mappings()); - } - public boolean hasMappings() { return !mappings().isEmpty(); } @@ -144,7 +138,7 @@ public PropertyMappings mergeWith(PropertyMappings other) { return other; } if (!other.hasMappings()) { - return PropertyMappings.copyOf(this); + return ImmutablePropertyMappings.copyOf(this); } Builder builder = PropertyMappings.builder(); builder.addMappings(Stream.concat(stream(), other.stream()).distinct()); @@ -168,7 +162,7 @@ public static Builder builder() { } @AccessibleFields - public static final class Builder extends PropertyMappings.Builder { + public static final class Builder extends ImmutablePropertyMappings.Builder { private Aggregation aggregation; diff --git a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java index 218ec70b04e..424357c7ac1 100644 --- a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java +++ b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java @@ -33,15 +33,14 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.text.MatchesPattern.matchesPattern; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.neo4j.gds.AbstractNodeProjection.LABEL_KEY; import static org.neo4j.gds.ElementProjection.PROPERTIES_KEY; import static org.neo4j.gds.NodeLabel.ALL_NODES; +import static org.neo4j.gds.NodeProjection.LABEL_KEY; class NodeProjectionsTest { @@ -50,10 +49,10 @@ class NodeProjectionsTest { void syntacticSugars(Object argument) { NodeProjections actual = NodeProjections.fromObject(argument); - NodeProjections expected = NodeProjections.builder().projections(singletonMap( + NodeProjections expected = NodeProjections.single( NodeLabel.of("A"), NodeProjection.builder().label("A").properties(PropertyMappings.of()).build() - )).build(); + ); assertThat( actual, @@ -73,19 +72,17 @@ void shouldParseWithProperties() { ) )); - NodeProjections expected = NodeProjections.builder().projections(singletonMap( + NodeProjections expected = NodeProjections.single( NodeLabel.of("MY_LABEL"), NodeProjection .builder() .label("A") - .properties(PropertyMappings - .builder() - .addMapping(PropertyMapping.of("prop1", DefaultValue.DEFAULT)) - .addMapping(PropertyMapping.of("prop2", DefaultValue.DEFAULT)) - .build() + .addProperties( + PropertyMapping.of("prop1", DefaultValue.DEFAULT), + PropertyMapping.of("prop2", DefaultValue.DEFAULT) ) .build() - )).build(); + ); assertThat( actual, @@ -98,10 +95,10 @@ void shouldParseWithProperties() { void shouldParseMultipleLabels() { NodeProjections actual = NodeProjections.fromObject(Arrays.asList("A", "B")); - NodeProjections expected = NodeProjections.builder() - .putProjection(NodeLabel.of("A"), NodeProjection.builder().label("A").build()) - .putProjection(NodeLabel.of("B"), NodeProjection.builder().label("B").build()) - .build(); + NodeProjections expected = NodeProjections.create(Map.of( + NodeLabel.of("A"), NodeProjection.builder().label("A").build(), + NodeLabel.of("B"), NodeProjection.builder().label("B").build() + )); assertThat(actual, equalTo(expected)); assertThat(actual.labelProjection(), equalTo("A, B")); @@ -111,14 +108,14 @@ void shouldParseMultipleLabels() { void shouldSupportStar() { NodeProjections actual = NodeProjections.fromObject("*"); - NodeProjections expected = NodeProjections.builder().projections(singletonMap( + NodeProjections expected = NodeProjections.single( ALL_NODES, NodeProjection .builder() .label("*") .properties(PropertyMappings.of()) .build() - )).build(); + ); assertThat( actual, @@ -147,37 +144,69 @@ void shouldFailOnDuplicatePropertyKeys() { .hasMessage("Duplicate property key `prop`"); } - @Test - void shouldFailOnAmbiguousNodePropertyDefinition() { - var builder = NodeProjections.builder().projections(Map.of( + static Stream ambiguousPropertyMappings() { + return Stream.of( + Arguments.of( + "different neo key", + PropertyMapping.of("foo", "bar", DefaultValue.DEFAULT), + PropertyMapping.of("foo", "baz", DefaultValue.DEFAULT), + new String[]{ + "Specifying multiple neoPropertyKeys for the same property is not allowed, found propertyKey: `foo` with conflicting neoPropertyKeys:", + "`bar`", + "`baz`" + } + ), + Arguments.of( + "different default value types", + PropertyMapping.of("foo", "baz", DefaultValue.forLong()), + PropertyMapping.of("foo", "baz", DefaultValue.forDouble()), + new String[]{ + "Specifying different default values for the same property with identical neoPropertyKey is not allowed, found propertyKey: `foo` with conflicting default values:", + "`-9223372036854775808`", + "`NaN`", + } + ), + Arguments.of( + "different default values", + PropertyMapping.of("foo", "baz", DefaultValue.of(42)), + PropertyMapping.of("foo", "baz", DefaultValue.of(1337)), + new String[]{ + "Specifying different default values for the same property with identical neoPropertyKey is not allowed, found propertyKey: `foo` with conflicting default values:", + "`42`", + "`1337`", + } + ) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("ambiguousPropertyMappings") + void shouldFailOnAmbiguousNodePropertyDefinition( + String ignore, + PropertyMapping first, + PropertyMapping second, + String... messages + ) { + var builder = ImmutableNodeProjections.builder().projections(Map.of( NodeLabel.of("A"), NodeProjection .builder() .label("A") - .properties(PropertyMappings - .builder() - .addMapping(PropertyMapping.of("foo", "bar", DefaultValue.DEFAULT)) - .build() - ) + .addProperty(first) .build(), - NodeLabel.of("B"), NodeProjection .builder() .label("B") - .properties(PropertyMappings - .builder() - .addMapping(PropertyMapping.of("foo", "baz", DefaultValue.DEFAULT)) - .build() - ) + .addProperty(second) .build() - )); - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Specifying multiple neoPropertyKeys for the same property is not allowed, found propertyKey: foo, neoPropertyKeys"); + var throwableAssert = assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class); + for (var message : messages) { + throwableAssert.hasMessageContaining(message); + } } @Test @@ -191,19 +220,17 @@ void shouldSupportCaseInsensitiveConfigKeys() { ) )); - NodeProjections expected = NodeProjections.builder().projections(singletonMap( + NodeProjections expected = NodeProjections.single( NodeLabel.of("MY_LABEL"), NodeProjection .builder() .label("A") - .properties(PropertyMappings - .builder() - .addMapping(PropertyMapping.of("prop1", DefaultValue.DEFAULT)) - .addMapping(PropertyMapping.of("prop2", DefaultValue.DEFAULT)) - .build() + .addProperties( + PropertyMapping.of("prop1", DefaultValue.DEFAULT), + PropertyMapping.of("prop2", DefaultValue.DEFAULT) ) .build() - )).build(); + ); assertThat( actual, diff --git a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java index df488d04e11..00ed2f9cc66 100644 --- a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java +++ b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java @@ -33,7 +33,7 @@ import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.loading.GraphStoreBuilder; -import org.neo4j.gds.core.loading.Nodes; +import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; @@ -124,7 +124,7 @@ public double evaluate(EvaluationContext context) { .databaseId(inputGraphStore.databaseId()) .capabilities(inputGraphStore.capabilities()) .schema(filteredSchema) - .nodes(Nodes.of(idMap, nodePropertyStore)) + .nodes(ImmutableNodes.of(filteredSchema.nodeSchema(), idMap, nodePropertyStore)) .relationshipImportResult(relationshipImportResult) .concurrency(config.concurrency()) .build(); diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java index bff6c8a7062..0d5996c5843 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java @@ -27,116 +27,72 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -public abstract class ElementSchema< - SELF extends ElementSchema, - ELEMENT_IDENTIFIER extends ElementIdentifier, - ENTRY extends ElementSchemaEntry, - PROPERTY_SCHEMA extends PropertySchema - > { +public interface ElementSchema< + SELF extends ElementSchema, + ELEMENT_IDENTIFIER extends ElementIdentifier, + ENTRY extends ElementSchemaEntry, + PROPERTY_SCHEMA extends PropertySchema> { - protected final Map entries; - ElementSchema(Map entries) { - this.entries = entries; - } - - abstract SELF filter(Set elementIdentifiersToKeep); + SELF filter(Set elementIdentifiersToKeep); - abstract SELF union(SELF other); - - public Collection entries() { - return this.entries.values(); - } + SELF union(SELF other); - public void set(ENTRY entry) { - entries.put(entry.identifier(), entry); - } - public ENTRY get(ELEMENT_IDENTIFIER identifier) { - return entries.get(identifier); - } + Collection entries(); - public void remove(ELEMENT_IDENTIFIER identifier) { - entries.remove(identifier); - } + ENTRY get(ELEMENT_IDENTIFIER identifier); - public Set allProperties() { - return this.entries - .values() + default Set allProperties() { + return entries() .stream() .flatMap(entry -> entry.properties().keySet().stream()) .collect(Collectors.toSet()); } - public Set allProperties(ELEMENT_IDENTIFIER elementIdentifier) { - return Optional.ofNullable(this.entries.get(elementIdentifier)) + default Set allProperties(ELEMENT_IDENTIFIER elementIdentifier) { + return Optional.ofNullable(get(elementIdentifier)) .map(entry -> entry.properties().keySet()) .orElse(Set.of()); } - public boolean hasProperties() { - return entries.values().stream().anyMatch(entry -> !entry.properties().isEmpty()); + default boolean hasProperties() { + return entries().stream().anyMatch(entry -> !entry.properties().isEmpty()); } - public boolean hasProperty(ELEMENT_IDENTIFIER elementIdentifier, String propertyKey) { - return entries.containsKey(elementIdentifier) && entries - .get(elementIdentifier) - .properties() - .containsKey(propertyKey); + default boolean hasProperty(ELEMENT_IDENTIFIER elementIdentifier, String propertyKey) { + return Optional.ofNullable(get(elementIdentifier)) + .map(entry -> entry.properties().containsKey(propertyKey)) + .orElse(false); } - public List propertySchemasFor(ELEMENT_IDENTIFIER elementIdentifier) { + default List propertySchemasFor(ELEMENT_IDENTIFIER elementIdentifier) { return Optional - .ofNullable(entries.get(elementIdentifier)) + .ofNullable(get(elementIdentifier)) .map(entry -> entry.properties().values()) .map(ArrayList::new) .orElse(new ArrayList<>()); } - /** - * Returns a union of all properties in the given schema. - */ - public Map unionProperties() { - return entries - .values() + default Map unionProperties() { + return entries() .stream() .flatMap(e -> e.properties().entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (leftSchema, rightSchema) -> leftSchema)); } - - public Map toMap() { - return entries - .entrySet() + default Map toMap() { + return entries() .stream() - .collect(Collectors.toMap(entry -> entry.getKey().name, entry -> entry.getValue().toMap())); + .collect(Collectors.toMap(entry -> entry.identifier().name, ElementSchemaEntry::toMap)); } - Map unionEntries(SELF other) { + default Map unionEntries(SELF other) { return Stream - .concat(entries.entrySet().stream(), other.entries.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, ElementSchemaEntry::union)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ElementSchema that = (ElementSchema) o; - - return entries.equals(that.entries); - } - - @Override - public int hashCode() { - return entries.hashCode(); - } - - @Override - public String toString() { - return toMap().toString(); + .concat(entries().stream(), other.entries().stream()) + .collect(Collectors.toMap(ElementSchemaEntry::identifier, Function.identity(), ElementSchemaEntry::union)); } } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchemaEntry.java index 365fd97ef88..284a31388a7 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchemaEntry.java @@ -28,29 +28,17 @@ import static java.lang.String.format; -public abstract class ElementSchemaEntry, ELEMENT_IDENTIFIER extends ElementIdentifier, PROPERTY_SCHEMA extends PropertySchema> { +public interface ElementSchemaEntry, ELEMENT_IDENTIFIER extends ElementIdentifier, PROPERTY_SCHEMA extends PropertySchema> { - public final ELEMENT_IDENTIFIER identifier; - public final Map properties; + ELEMENT_IDENTIFIER identifier(); - public ElementSchemaEntry(ELEMENT_IDENTIFIER identifier, Map properties) { - this.identifier = identifier; - this.properties = properties; - } - - public ELEMENT_IDENTIFIER identifier() { - return identifier; - } + Map properties(); - public Map properties() { - return properties; - } + SELF union(SELF other); - abstract SELF union(SELF other); + Map toMap(); - abstract Map toMap(); - - protected Map unionProperties(Map rightProperties) { + default Map unionProperties(Map rightProperties) { return Stream .concat(properties().entrySet().stream(), rightProperties.entrySet().stream()) .collect(Collectors.toMap( @@ -58,7 +46,7 @@ protected Map unionProperties(Map entry.getValue().valueType())), @@ -72,27 +60,4 @@ protected Map unionProperties(Map that = (ElementSchemaEntry) o; - - if (!identifier.equals(that.identifier)) return false; - return properties.equals(that.properties); - } - - @Override - public int hashCode() { - int result = identifier.hashCode(); - result = 31 * result + properties.hashCode(); - return result; - } - - @Override - public String toString() { - return toMap().toString(); - } } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java index 804cabc2dc9..4d0059f5ace 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java @@ -21,15 +21,12 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.annotation.ValueClass; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; -@ValueClass public interface GraphSchema { NodeSchema nodeSchema(); @@ -38,6 +35,12 @@ public interface GraphSchema { Map graphProperties(); + GraphSchema filterNodeLabels(Set labelsToKeep); + + GraphSchema filterRelationshipTypes(Set relationshipTypesToKeep); + + GraphSchema union(GraphSchema other); + default Map toMap() { return Map.of( "nodes", nodeSchema().toMap(), @@ -60,50 +63,20 @@ default Map toMapOld() { ); } - default GraphSchema filterNodeLabels(Set labelsToKeep) { - return of(nodeSchema().filter(labelsToKeep), relationshipSchema(), graphProperties()); - } - - default GraphSchema filterRelationshipTypes(Set relationshipTypesToKeep) { - return of(nodeSchema(), relationshipSchema().filter(relationshipTypesToKeep), graphProperties()); + default boolean isUndirected() { + return relationshipSchema().isUndirected(); } - default GraphSchema union(GraphSchema other) { - return GraphSchema.of( - nodeSchema().union(other.nodeSchema()), - relationshipSchema().union(other.relationshipSchema()), - unionGraphProperties(other.graphProperties()) - ); + default Direction direction() { + return relationshipSchema().isUndirected() ? Direction.UNDIRECTED : Direction.DIRECTED; } - private Map unionGraphProperties(Map otherProperties) { - return Stream.concat( - graphProperties().entrySet().stream(), - otherProperties.entrySet().stream() - ).collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue, - (leftType, rightType) -> { - if (leftType.valueType() != rightType.valueType()) { - throw new IllegalArgumentException(String.format( - Locale.ENGLISH, - "Combining schema entries with value type %s and %s is not supported.", - leftType.valueType(), - rightType.valueType() - )); - } else { - return leftType; - } - } - )); + static GraphSchema empty() { + return MutableGraphSchema.empty(); } - static GraphSchema of(NodeSchema nodeSchema, RelationshipSchema relationshipSchema, Map graphProperties) { - return ImmutableGraphSchema.builder() - .nodeSchema(nodeSchema) - .relationshipSchema(relationshipSchema) - .graphProperties(graphProperties) - .build(); + static MutableGraphSchema mutable() { + return MutableGraphSchema.empty(); } static String forPropertySchema(PS propertySchema) { @@ -125,15 +98,4 @@ static String forPropertySchema(PS propertySchema) { ); } - static GraphSchema empty() { - return of(NodeSchema.empty(), RelationshipSchema.empty(), Map.of()); - } - - default boolean isUndirected() { - return relationshipSchema().isUndirected(); - } - - default Direction direction() { - return relationshipSchema().isUndirected() ? Direction.UNDIRECTED : Direction.DIRECTED; - } } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java new file mode 100644 index 00000000000..ff5093ada1d --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.annotation.ValueClass; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@ValueClass +public interface MutableGraphSchema extends GraphSchema { + @Override + MutableNodeSchema nodeSchema(); + + @Override + MutableRelationshipSchema relationshipSchema(); + + + @Override + default MutableGraphSchema filterNodeLabels(Set labelsToKeep) { + return of(nodeSchema().filter(labelsToKeep), relationshipSchema(), graphProperties()); + } + + @Override + default MutableGraphSchema filterRelationshipTypes(Set relationshipTypesToKeep) { + return of(nodeSchema(), relationshipSchema().filter(relationshipTypesToKeep), graphProperties()); + } + + @Override + default MutableGraphSchema union(GraphSchema other) { + return MutableGraphSchema.of( + nodeSchema().union(other.nodeSchema()), + relationshipSchema().union(other.relationshipSchema()), + unionGraphProperties(other.graphProperties()) + ); + } + + static MutableGraphSchema empty() { + return of(MutableNodeSchema.empty(), MutableRelationshipSchema.empty(), Map.of()); + } + + static MutableGraphSchema from(GraphSchema from) { + return of( + MutableNodeSchema.from(from.nodeSchema()), + MutableRelationshipSchema.from(from.relationshipSchema()), + new HashMap<>(from.graphProperties()) + ); + } + + static MutableGraphSchema of( + MutableNodeSchema nodeSchema, + MutableRelationshipSchema relationshipSchema, + Map graphProperties + ) { + return ImmutableMutableGraphSchema.builder() + .nodeSchema(nodeSchema) + .relationshipSchema(relationshipSchema) + .graphProperties(graphProperties) + .build(); + } + + static ImmutableMutableGraphSchema.Builder builder() { + return ImmutableMutableGraphSchema.builder(); + } + + private Map unionGraphProperties(Map otherProperties) { + return Stream.concat( + graphProperties().entrySet().stream(), + otherProperties.entrySet().stream() + ).collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (leftType, rightType) -> { + if (leftType.valueType() != rightType.valueType()) { + throw new IllegalArgumentException(String.format( + Locale.ENGLISH, + "Combining schema entries with value type %s and %s is not supported.", + leftType.valueType(), + rightType.valueType() + )); + } else { + return leftType; + } + } + )); + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java new file mode 100644 index 00000000000..af9603e67a0 --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public final class MutableNodeSchema implements NodeSchema { + + private final Map entries; + + public static MutableNodeSchema empty() { + return new MutableNodeSchema(new LinkedHashMap<>()); + } + + private MutableNodeSchema(Map entries) { + this.entries=entries; + } + + public static MutableNodeSchema from(NodeSchema fromSchema) { + var nodeSchema = MutableNodeSchema.empty(); + fromSchema.entries().forEach(fromEntry -> nodeSchema.set(MutableNodeSchemaEntry.from(fromEntry))); + + return nodeSchema; + } + + @Override + public Set availableLabels() { + return entries.keySet(); + } + + @Override + public boolean containsOnlyAllNodesLabel() { + return availableLabels().size() == 1 && availableLabels().contains(NodeLabel.ALL_NODES); + } + + @Override + public MutableNodeSchema filter(Set labelsToKeep) { + return new MutableNodeSchema(entries + .entrySet() + .stream() + .filter(e -> labelsToKeep.contains(e.getKey())) + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> MutableNodeSchemaEntry.from(entry.getValue()) + ))); + } + + @Override + public MutableNodeSchema union(NodeSchema other) { + return new MutableNodeSchema(unionEntries(other)); + } + + @Override + public Collection entries() { + return entries.values(); + } + + @Override + public MutableNodeSchemaEntry get(NodeLabel identifier) { + return entries.get(identifier); + } + + public void set(MutableNodeSchemaEntry entry) { + entries.put(entry.identifier(), entry); + } + + public void remove(NodeLabel identifier) { + entries.remove(identifier); + } + + public MutableNodeSchemaEntry getOrCreateLabel(NodeLabel key) { + return this.entries.computeIfAbsent(key, (__) -> new MutableNodeSchemaEntry(key)); + } + + public MutableNodeSchema addLabel(NodeLabel nodeLabel) { + getOrCreateLabel(nodeLabel); + return this; + } + + public MutableNodeSchema addLabel(NodeLabel nodeLabel, Map nodeProperties) { + var nodeSchemaEntry = getOrCreateLabel(nodeLabel); + nodeProperties.forEach(nodeSchemaEntry::addProperty); + return this; + } + + public MutableNodeSchema addProperty(NodeLabel nodeLabel, String propertyName, PropertySchema propertySchema) { + getOrCreateLabel(nodeLabel).addProperty(propertyName, propertySchema); + return this; + } + + public MutableNodeSchema addProperty(NodeLabel nodeLabel, String propertyKey, ValueType valueType) { + getOrCreateLabel(nodeLabel).addProperty(propertyKey, valueType); + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MutableNodeSchema that = (MutableNodeSchema) o; + + return entries.equals(that.entries); + } + + @Override + public int hashCode() { + return entries.hashCode(); + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchemaEntry.java new file mode 100644 index 00000000000..90a2b208bab --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchemaEntry.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public class MutableNodeSchemaEntry implements NodeSchemaEntry { + + private final NodeLabel nodeLabel; + private final Map properties; + + static MutableNodeSchemaEntry from(NodeSchemaEntry fromEntry) { + return new MutableNodeSchemaEntry(fromEntry.identifier(), new HashMap<>(fromEntry.properties())); + } + + MutableNodeSchemaEntry(NodeLabel identifier) { + this(identifier, new LinkedHashMap<>()); + } + + public MutableNodeSchemaEntry(NodeLabel nodeLabel, Map properties) { + this.nodeLabel = nodeLabel; + this.properties = properties; + } + + @Override + public NodeLabel identifier() { + return nodeLabel; + } + + @Override + public Map properties() { + return Map.copyOf(properties); + } + + @Override + public MutableNodeSchemaEntry union(MutableNodeSchemaEntry other) { + if (!other.identifier().equals(this.identifier())) { + throw new UnsupportedOperationException( + formatWithLocale( + "Cannot union node schema entries with different node labels %s and %s", + this.identifier(), + other.identifier() + ) + ); + } + + return new MutableNodeSchemaEntry(this.identifier(), unionProperties(other.properties)); + } + + @Override + public Map toMap() { + return properties + .entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) + ) + + ); + } + + public MutableNodeSchemaEntry addProperty(String propertyName, ValueType valueType) { + return addProperty( + propertyName, + PropertySchema.of( + propertyName, + valueType, + valueType.fallbackValue(), + PropertyState.PERSISTENT + ) + ); + } + + public MutableNodeSchemaEntry addProperty(String propertyName, PropertySchema propertySchema) { + this.properties.put(propertyName, propertySchema); + return this; + } + + public void removeProperty(String propertyName) { + properties.remove(propertyName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MutableNodeSchemaEntry that = (MutableNodeSchemaEntry) o; + + if (!nodeLabel.equals(that.nodeLabel)) return false; + return properties.equals(that.properties); + } + + @Override + public int hashCode() { + int result = nodeLabel.hashCode(); + result = 31 * result + properties.hashCode(); + return result; + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java new file mode 100644 index 00000000000..3f5d12f4e91 --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java @@ -0,0 +1,174 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class MutableRelationshipSchema implements RelationshipSchema { + + private final Map entries; + + public static MutableRelationshipSchema empty() { + return new MutableRelationshipSchema(new LinkedHashMap<>()); + } + + public MutableRelationshipSchema(Map entries) { + this.entries = entries; + } + + public static MutableRelationshipSchema from(RelationshipSchema fromSchema) { + var relationshipSchema = MutableRelationshipSchema.empty(); + fromSchema.entries().forEach(fromEntry -> relationshipSchema.set(MutableRelationshipSchemaEntry.from(fromEntry))); + + return relationshipSchema; + } + + @Override + public MutableRelationshipSchema filter(Set relationshipTypesToKeep) { + return new MutableRelationshipSchema(entries + .entrySet() + .stream() + .filter(e -> relationshipTypesToKeep.contains(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> MutableRelationshipSchemaEntry.from(entry.getValue())))); + } + + @Override + public MutableRelationshipSchema union(RelationshipSchema other) { + return new MutableRelationshipSchema(unionEntries(other)); + } + + @Override + public Collection entries() { + return entries.values(); + } + + @Override + public MutableRelationshipSchemaEntry get(RelationshipType identifier) { + return entries.get(identifier); + } + + @Override + public Set availableTypes() { + return entries.keySet(); + } + + @Override + public boolean isUndirected() { + // a graph with no relationships is considered undirected + // this is because algorithms such as TriangleCount are still well-defined + // so it is the least restrictive decision + return entries.values().stream().allMatch(MutableRelationshipSchemaEntry::isUndirected); + } + + @Override + public boolean isUndirected(RelationshipType type) { + return entries.get(type).isUndirected(); + } + + // TODO: remove + @Override + public Map directions() { + return availableTypes() + .stream() + .collect(Collectors.toMap( + relationshipType -> relationshipType, + relationshipType -> entries.get(relationshipType).direction() + )); + } + + @Override + public Object toMapOld() { + return entries() + .stream() + .collect(Collectors.toMap(e -> e.identifier().name(), + e -> e + .properties() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) + )) + )); + } + + public void set(MutableRelationshipSchemaEntry entry) { + entries.put(entry.identifier(), entry); + } + public void remove(RelationshipType identifier) { + entries.remove(identifier); + } + + public MutableRelationshipSchemaEntry getOrCreateRelationshipType( + RelationshipType relationshipType, Direction direction + ) { + return this.entries.computeIfAbsent(relationshipType, + __ -> new MutableRelationshipSchemaEntry(relationshipType, direction) + ); + } + + public MutableRelationshipSchema addRelationshipType(RelationshipType relationshipType, Direction direction) { + getOrCreateRelationshipType(relationshipType, direction); + return this; + } + + public MutableRelationshipSchema addProperty( + RelationshipType relationshipType, + Direction direction, + String propertyKey, + RelationshipPropertySchema propertySchema + ) { + getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, propertySchema); + return this; + } + + public MutableRelationshipSchema addProperty( + RelationshipType relationshipType, + Direction direction, + String propertyKey, + ValueType valueType, + PropertyState propertyState + ) { + getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, valueType, propertyState); + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MutableRelationshipSchema that = (MutableRelationshipSchema) o; + + return entries.equals(that.entries); + } + + @Override + public int hashCode() { + return entries.hashCode(); + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java new file mode 100644 index 00000000000..64c12948339 --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java @@ -0,0 +1,154 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.nodeproperties.ValueType; +import org.neo4j.gds.core.Aggregation; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public class MutableRelationshipSchemaEntry implements RelationshipSchemaEntry { + + private final RelationshipType relationshipType; + private final Direction direction; + private final Map properties; + + public MutableRelationshipSchemaEntry(RelationshipType identifier, Direction direction) { + this(identifier, direction, new LinkedHashMap<>()); + } + + public MutableRelationshipSchemaEntry( + RelationshipType relationshipType, + Direction direction, + Map properties + ) { + this.relationshipType = relationshipType; + this.direction = direction; + this.properties = properties; + } + + public static MutableRelationshipSchemaEntry from(RelationshipSchemaEntry fromEntry) { + return new MutableRelationshipSchemaEntry( + fromEntry.identifier(), + fromEntry.direction(), + new HashMap<>(fromEntry.properties()) + ); + } + + @Override + public Direction direction() { + return this.direction; + } + + @Override + public boolean isUndirected() { + return direction() == Direction.UNDIRECTED; + } + + @Override + public RelationshipType identifier() { + return relationshipType; + } + + @Override + public Map properties() { + return Map.copyOf(properties); + } + + @Override + public MutableRelationshipSchemaEntry union(MutableRelationshipSchemaEntry other) { + if (!other.identifier().equals(this.identifier())) { + throw new UnsupportedOperationException( + formatWithLocale( + "Cannot union relationship schema entries with different types %s and %s", + this.identifier(), + other.identifier() + ) + ); + } + + if (other.isUndirected() != this.isUndirected()) { + throw new IllegalArgumentException( + formatWithLocale("Conflicting directionality for relationship types %s", this.identifier().name)); + } + + return new MutableRelationshipSchemaEntry(this.identifier(), this.direction, unionProperties(other.properties)); + } + + @Override + public Map toMap() { + return Map.of( + "direction", direction.name(), + "properties", properties + .entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) + ) + ) + ); + } + + public MutableRelationshipSchemaEntry addProperty(String propertyKey, ValueType valueType, PropertyState propertyState) { + return addProperty( + propertyKey, + RelationshipPropertySchema.of( + propertyKey, + valueType, + valueType.fallbackValue(), + propertyState, + Aggregation.DEFAULT + ) + ); + } + + public MutableRelationshipSchemaEntry addProperty(String propertyKey, RelationshipPropertySchema propertySchema) { + this.properties.put(propertyKey, propertySchema); + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MutableRelationshipSchemaEntry that = (MutableRelationshipSchemaEntry) o; + + if (!relationshipType.equals(that.relationshipType)) return false; + if (direction != that.direction) return false; + return properties.equals(that.properties); + } + + @Override + public int hashCode() { + int result = relationshipType.hashCode(); + result = 31 * result + direction.hashCode(); + result = 31 * result + properties.hashCode(); + return result; + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java index 712a3a84109..337395b2ccb 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java @@ -20,77 +20,13 @@ package org.neo4j.gds.api.schema; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.api.nodeproperties.ValueType; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; -public final class NodeSchema extends ElementSchema { +public interface NodeSchema extends ElementSchema { + Set availableLabels(); - public static NodeSchema empty() { - return new NodeSchema(new LinkedHashMap<>()); - } + boolean containsOnlyAllNodesLabel(); - private NodeSchema(Map entries) { - super(entries); - } - - public static NodeSchema from(NodeSchema fromSchema) { - var nodeSchema = NodeSchema.empty(); - fromSchema.entries().forEach(fromEntry -> nodeSchema.set(NodeSchemaEntry.from(fromEntry))); - - return nodeSchema; - } - - public Set availableLabels() { - return entries.keySet(); - } - - public boolean containsOnlyAllNodesLabel() { - return availableLabels().size() == 1 && availableLabels().contains(NodeLabel.ALL_NODES); - } - - public NodeSchema filter(Set labelsToKeep) { - return new NodeSchema(entries - .entrySet() - .stream() - .filter(e -> labelsToKeep.contains(e.getKey())) - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> NodeSchemaEntry.from(entry.getValue()) - ))); - } - - @Override - public NodeSchema union(NodeSchema other) { - return new NodeSchema(unionEntries(other)); - } - - - public NodeSchemaEntry getOrCreateLabel(NodeLabel key) { - return this.entries.computeIfAbsent(key, (__) -> new NodeSchemaEntry(key)); - } - - public NodeSchema addLabel(NodeLabel nodeLabel) { - getOrCreateLabel(nodeLabel); - return this; - } - - public NodeSchema addLabel(NodeLabel nodeLabel, Map nodeProperties) { - var nodeSchemaEntry = getOrCreateLabel(nodeLabel); - nodeProperties.forEach(nodeSchemaEntry::addProperty); - return this; - } - - public NodeSchema addProperty(NodeLabel nodeLabel, String propertyName, PropertySchema propertySchema) { - getOrCreateLabel(nodeLabel).addProperty(propertyName, propertySchema); - return this; - } - - public NodeSchema addProperty(NodeLabel nodeLabel, String propertyKey, ValueType valueType) { - getOrCreateLabel(nodeLabel).addProperty(propertyKey, valueType); - return this; - } + NodeSchema filter(Set labelsToKeep); } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchemaEntry.java index cdd772d2c07..ffde9858669 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchemaEntry.java @@ -20,76 +20,7 @@ package org.neo4j.gds.api.schema; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.nodeproperties.ValueType; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.stream.Collectors; +public interface NodeSchemaEntry extends ElementSchemaEntry { -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; - -public class NodeSchemaEntry extends ElementSchemaEntry { - - NodeSchemaEntry(NodeLabel identifier) { - this(identifier, new LinkedHashMap<>()); - } - - public NodeSchemaEntry(NodeLabel identifier, Map properties) { - super(identifier, properties); - } - - static NodeSchemaEntry from(NodeSchemaEntry fromEntry) { - return new NodeSchemaEntry(fromEntry.identifier(), new HashMap<>(fromEntry.properties())); - } - - @Override - NodeSchemaEntry union(NodeSchemaEntry other) { - if (!other.identifier().equals(this.identifier())) { - throw new UnsupportedOperationException( - formatWithLocale( - "Cannot union node schema entries with different node labels %s and %s", - this.identifier(), - other.identifier() - ) - ); - } - - return new NodeSchemaEntry(this.identifier(), unionProperties(other.properties)); - } - - public NodeSchemaEntry addProperty(String propertyName, ValueType valueType) { - return addProperty( - propertyName, - PropertySchema.of( - propertyName, - valueType, - valueType.fallbackValue(), - PropertyState.PERSISTENT - ) - ); - } - - public NodeSchemaEntry addProperty(String propertyName, PropertySchema propertySchema) { - this.properties.put(propertyName, propertySchema); - return this; - } - - public void removeProperty(String propertyName) { - properties.remove(propertyName); - } - - @Override - Map toMap() { - return properties - .entrySet() - .stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) - ) - - ); - } } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java index a82f26ddc80..4c1d3e56416 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java @@ -20,116 +20,19 @@ package org.neo4j.gds.api.schema; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.nodeproperties.ValueType; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; +public interface RelationshipSchema extends ElementSchema { + Set availableTypes(); -public class RelationshipSchema extends ElementSchema { + boolean isUndirected(); - public static RelationshipSchema empty() { - return new RelationshipSchema(new LinkedHashMap<>()); - } - - public RelationshipSchema(Map entries) { - super(entries); - } - - public static RelationshipSchema from(RelationshipSchema fromSchema) { - var relationshipSchema = RelationshipSchema.empty(); - fromSchema.entries().forEach(fromEntry -> relationshipSchema.set(RelationshipSchemaEntry.from(fromEntry))); - - return relationshipSchema; - } - - @Override - public RelationshipSchema filter(Set relationshipTypesToKeep) { - return new RelationshipSchema(entries - .entrySet() - .stream() - .filter(e -> relationshipTypesToKeep.contains(e.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, entry -> RelationshipSchemaEntry.from(entry.getValue())))); - } - - @Override - public RelationshipSchema union(RelationshipSchema other) { - return new RelationshipSchema(unionEntries(other)); - } - - public Set availableTypes() { - return entries.keySet(); - } - - public boolean isUndirected() { - // a graph with no relationships is considered undirected - // this is because algorithms such as TriangleCount are still well-defined - // so it is the least restrictive decision - return entries.values().stream().allMatch(RelationshipSchemaEntry::isUndirected); - } - - public boolean isUndirected(RelationshipType type) { - return entries.get(type).isUndirected(); - } - - public RelationshipSchemaEntry getOrCreateRelationshipType( - RelationshipType relationshipType, Direction direction - ) { - return this.entries.computeIfAbsent(relationshipType, - __ -> new RelationshipSchemaEntry(relationshipType, direction) - ); - } - - public RelationshipSchema addRelationshipType(RelationshipType relationshipType, Direction direction) { - getOrCreateRelationshipType(relationshipType, direction); - return this; - } - - public RelationshipSchema addProperty( - RelationshipType relationshipType, - Direction direction, - String propertyKey, - RelationshipPropertySchema propertySchema - ) { - getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, propertySchema); - return this; - } - - public RelationshipSchema addProperty( - RelationshipType relationshipType, - Direction direction, - String propertyKey, - ValueType valueType, - PropertyState propertyState - ) { - getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, valueType, propertyState); - return this; - } + boolean isUndirected(RelationshipType type); // TODO: remove - public Map directions() { - return availableTypes() - .stream() - .collect(Collectors.toMap( - relationshipType -> relationshipType, - relationshipType -> entries.get(relationshipType).direction() - )); - } + Map directions(); - Object toMapOld() { - return entries() - .stream() - .collect(Collectors.toMap(e -> e.identifier().name(), - e -> e - .properties() - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, - innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) - )) - )); - } + Object toMapOld(); } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java index d468a96b722..406c1d776f6 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java @@ -20,118 +20,16 @@ package org.neo4j.gds.api.schema; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.core.Aggregation; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.stream.Collectors; +public interface RelationshipSchemaEntry extends ElementSchemaEntry { + Direction direction(); -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + boolean isUndirected(); -public class RelationshipSchemaEntry extends ElementSchemaEntry { - - private final Direction direction; - - RelationshipSchemaEntry(RelationshipType identifier, Direction direction) { - this(identifier, direction, new LinkedHashMap<>()); - } - - public RelationshipSchemaEntry( - RelationshipType identifier, - Direction direction, - Map properties + static RelationshipSchemaEntry empty( + RelationshipType relationshipType, + Direction direction ) { - super(identifier, properties); - this.direction = direction; - } - - static RelationshipSchemaEntry from(RelationshipSchemaEntry fromEntry) { - return new RelationshipSchemaEntry( - fromEntry.identifier(), - fromEntry.direction, - new HashMap<>(fromEntry.properties()) - ); - } - - public Direction direction() { - return this.direction; - } - - boolean isUndirected() { - return direction() == Direction.UNDIRECTED; - } - - @Override - RelationshipSchemaEntry union(RelationshipSchemaEntry other) { - if (!other.identifier().equals(this.identifier())) { - throw new UnsupportedOperationException( - formatWithLocale( - "Cannot union relationship schema entries with different types %s and %s", - this.identifier(), - other.identifier() - ) - ); - } - - if (other.isUndirected() != this.isUndirected()) { - throw new IllegalArgumentException( - formatWithLocale("Conflicting directionality for relationship types %s", this.identifier().name)); - } - - return new RelationshipSchemaEntry(this.identifier(), this.direction, unionProperties(other.properties)); - } - - public RelationshipSchemaEntry addProperty(String propertyKey, ValueType valueType, PropertyState propertyState) { - return addProperty( - propertyKey, - RelationshipPropertySchema.of( - propertyKey, - valueType, - valueType.fallbackValue(), - propertyState, - Aggregation.DEFAULT - ) - ); - } - - public RelationshipSchemaEntry addProperty(String propertyKey, RelationshipPropertySchema propertySchema) { - this.properties.put(propertyKey, propertySchema); - return this; - } - - @Override - Map toMap() { - return Map.of( - "direction", direction.name(), - "properties", properties - .entrySet() - .stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) - ) - ) - ); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - RelationshipSchemaEntry that = (RelationshipSchemaEntry) o; - - return direction == that.direction; - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + direction.hashCode(); - return result; + return new MutableRelationshipSchemaEntry(relationshipType, direction); } } diff --git a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/GraphSchemaTest.java b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/GraphSchemaTest.java index 1e7d94ba085..5c4159aec51 100644 --- a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/GraphSchemaTest.java +++ b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/GraphSchemaTest.java @@ -36,21 +36,21 @@ void testUnionOfGraphPropertiesForDifferentProperties() { var prop1Schema = PropertySchema.of("prop1", ValueType.LONG, DefaultValue.of(42L), PropertyState.TRANSIENT); var prop2Schema = PropertySchema.of("prop2", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.TRANSIENT); - var graphSchema1 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema1 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema) ); - var graphSchema2 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema2 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop2", prop2Schema) ); - var graphSchemaUnion = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchemaUnion = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of( "prop1", prop1Schema, "prop2", prop2Schema @@ -64,15 +64,15 @@ void testUnionOfGraphPropertiesForDifferentProperties() { void testUnionOfGraphPropertiesForSameProperties() { var prop1Schema = PropertySchema.of("prop1", ValueType.LONG, DefaultValue.of(42L), PropertyState.TRANSIENT); - var graphSchema1 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema1 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema) ); - var graphSchema2 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema2 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema) ); @@ -84,15 +84,15 @@ void testUnionOfGraphPropertiesForSamePropertiesWithDifferentTypeThrows() { var prop1Schema1 = PropertySchema.of("prop1", ValueType.LONG, DefaultValue.of(42L), PropertyState.TRANSIENT); var prop1Schema2 = PropertySchema.of("prop1", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.TRANSIENT); - var graphSchema1 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema1 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema1) ); - var graphSchema2 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema2 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema2) ); diff --git a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/NodeSchemaTest.java b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/NodeSchemaTest.java index 8776e4bdb6e..8b23148b83e 100644 --- a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/NodeSchemaTest.java +++ b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/NodeSchemaTest.java @@ -42,7 +42,7 @@ class NodeSchemaTest { @Test void handlesOutsideOfSchemaRequests() { - NodeSchema empty = NodeSchema.empty(); + MutableNodeSchema empty = MutableNodeSchema.empty(); assertFalse(empty.hasProperty(NodeLabel.of("NotInSchema"), "notInSchemaEither")); } @@ -52,7 +52,7 @@ void testDefaultValues() { DefaultValue defaultValue = DefaultValue.of(42.0D); String propertyName = "baz"; - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema .getOrCreateLabel(label) .addProperty( @@ -70,13 +70,13 @@ void testFiltering() { var label1 = NodeLabel.of("Foo"); var label2 = NodeLabel.of("Bar"); - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE).addProperty("baz", ValueType.DOUBLE); nodeSchema.getOrCreateLabel(label2).addProperty("baz", ValueType.DOUBLE); assertEquals(nodeSchema, nodeSchema.filter(Set.of(label1, label2))); - var expected = NodeSchema.empty(); + var expected = MutableNodeSchema.empty(); expected.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE).addProperty("baz", ValueType.DOUBLE); assertEquals(expected, nodeSchema.filter(Set.of(label1))); @@ -87,13 +87,13 @@ void testUnion() { var label1 = NodeLabel.of("Foo"); var label2 = NodeLabel.of("Bar"); - var nodeSchema1 = NodeSchema.empty(); + var nodeSchema1 = MutableNodeSchema.empty(); nodeSchema1.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE); - var nodeSchema2 = NodeSchema.empty(); + var nodeSchema2 = MutableNodeSchema.empty(); nodeSchema2.getOrCreateLabel(label2).addProperty("bar", ValueType.DOUBLE); - var expected = NodeSchema.empty(); + var expected = MutableNodeSchema.empty(); expected.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE); expected.getOrCreateLabel(label2).addProperty("bar", ValueType.DOUBLE); @@ -105,15 +105,15 @@ void testUnionSchema() { var label1 = NodeLabel.of("Foo"); var label2 = NodeLabel.of("Bar"); - var nodeSchema1 = NodeSchema.empty(); + var nodeSchema1 = MutableNodeSchema.empty(); nodeSchema1.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE); nodeSchema1.getOrCreateLabel(label2); - var nodeSchema2 = NodeSchema.empty(); + var nodeSchema2 = MutableNodeSchema.empty(); nodeSchema2.getOrCreateLabel(label1).addProperty("baz", ValueType.DOUBLE); nodeSchema2.getOrCreateLabel(label2).addProperty("baz", ValueType.DOUBLE); - var expected = NodeSchema.empty(); + var expected = MutableNodeSchema.empty(); expected.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE).addProperty("baz", ValueType.DOUBLE); expected.getOrCreateLabel(label2).addProperty("baz", ValueType.DOUBLE); @@ -124,10 +124,10 @@ void testUnionSchema() { void testUnionOfIncompatibleProperties() { var label1 = NodeLabel.of("Foo"); - var nodeSchema1 = NodeSchema.empty() + var nodeSchema1 = MutableNodeSchema.empty() .getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE); - var nodeSchema2 = NodeSchema.empty() + var nodeSchema2 = MutableNodeSchema.empty() .getOrCreateLabel(label1).addProperty("bar", ValueType.LONG); var ex = assertThrows(IllegalArgumentException.class, () -> nodeSchema1.union(nodeSchema2)); @@ -141,7 +141,7 @@ void testUnionProperties() { var label1 = NodeLabel.of("Foo"); var label2 = NodeLabel.of("Bar"); - var nodeSchema = NodeSchema.empty() + var nodeSchema = MutableNodeSchema.empty() .addProperty(label1, "foo", ValueType.DOUBLE) .addProperty(label1, "baz", ValueType.LONG) .addProperty(label2, "bar", ValueType.LONG_ARRAY) @@ -164,7 +164,7 @@ void testUnionProperties() { @Test void shouldCreateDeepCopiesWhenFiltering() { var nodeLabel = NodeLabel.of("A"); - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema.getOrCreateLabel(nodeLabel).addProperty("prop", ValueType.LONG); var filteredNodeSchema = nodeSchema.filter(Set.of(nodeLabel)); filteredNodeSchema @@ -176,17 +176,17 @@ void shouldCreateDeepCopiesWhenFiltering() { } static Stream schemaAndHasProperties() { - NodeSchema.empty(); + MutableNodeSchema.empty(); return Stream.of( - Arguments.of(NodeSchema.empty().addLabel(NodeLabel.of("A")), false), - Arguments.of(NodeSchema.empty().addProperty(NodeLabel.of("A"), "foo", ValueType.LONG), true) + Arguments.of(MutableNodeSchema.empty().addLabel(NodeLabel.of("A")), false), + Arguments.of(MutableNodeSchema.empty().addProperty(NodeLabel.of("A"), "foo", ValueType.LONG), true) ); } @ParameterizedTest @MethodSource("schemaAndHasProperties") - void shouldKnowIfPropertiesArePresent(NodeSchema nodeSchema, boolean hasProperties) { + void shouldKnowIfPropertiesArePresent(MutableNodeSchema nodeSchema, boolean hasProperties) { assertThat(nodeSchema.hasProperties()).isEqualTo(hasProperties); } @@ -194,7 +194,7 @@ void shouldKnowIfPropertiesArePresent(NodeSchema nodeSchema, boolean hasProperti void shouldAddNodeLabelWithNodeProperties() { var label = NodeLabel.of("Foo"); - var nodeSchema = NodeSchema.empty() + var nodeSchema = MutableNodeSchema.empty() .addProperty(label, "foo", ValueType.DOUBLE) .addProperty(label, "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT)) .addProperty(label, "bar", ValueType.LONG_ARRAY); diff --git a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java index a5f5065a73c..54fce9de052 100644 --- a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java +++ b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java @@ -41,31 +41,27 @@ class RelationshipSchemaTest { @Test void inAndOutUndirected() { - assertThat(RelationshipSchema.empty().isUndirected()).isTrue(); + assertThat(MutableRelationshipSchema.empty().isUndirected()).isTrue(); RelationshipType type = RelationshipType.of("TYPE"); - var propertySchema = Map.of( - type, - Map.of("prop", RelationshipPropertySchema.of("prop", ValueType.DOUBLE)) - ); - var undirectedSchema = RelationshipSchema.empty(); + var undirectedSchema = MutableRelationshipSchema.empty(); undirectedSchema.getOrCreateRelationshipType(type, Direction.UNDIRECTED); assertThat(undirectedSchema.isUndirected()).isTrue(); - var directedSchema = RelationshipSchema.empty(); + var directedSchema = MutableRelationshipSchema.empty(); directedSchema.getOrCreateRelationshipType(type, Direction.DIRECTED); assertThat(directedSchema.isUndirected()).isFalse(); } @Test void emptyIsUndirected() { - assertThat(RelationshipSchema.empty().isUndirected()).isTrue(); + assertThat(MutableRelationshipSchema.empty().isUndirected()).isTrue(); } @Test void handlesOutsideOfSchemaRequests() { - var empty = RelationshipSchema.empty(); + var empty = MutableRelationshipSchema.empty(); assertThat(empty.hasProperty(RelationshipType.of("NotInSchema"), "notInSchemaEither")).isFalse(); } @@ -77,7 +73,7 @@ void testDefaultValuesAndAggregation() { Aggregation aggregation = Aggregation.COUNT; PropertyState propertyState = PropertyState.PERSISTENT; String propertyName = "baz"; - var relationshipSchema = RelationshipSchema.empty() + var relationshipSchema = MutableRelationshipSchema.empty() .addProperty( relType, Direction.DIRECTED, @@ -115,14 +111,14 @@ void testFiltering(Direction orientation, boolean isUndirected) { var label1 = RelationshipType.of("Foo"); var label2 = RelationshipType.of("Bar"); - var relationshipSchema = RelationshipSchema.empty() + var relationshipSchema = MutableRelationshipSchema.empty() .addProperty(label1, orientation, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(label1, orientation, "baz", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(label2, orientation, "baz", ValueType.DOUBLE, PropertyState.PERSISTENT); assertThat(relationshipSchema.filter(Set.of(label1, label2))).isEqualTo(relationshipSchema); - var expected = RelationshipSchema.empty() + var expected = MutableRelationshipSchema.empty() .addProperty(label1, orientation, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(label1, orientation, "baz", ValueType.DOUBLE, PropertyState.PERSISTENT); @@ -135,9 +131,9 @@ void testFilteringMixed() { var directedType = RelationshipType.of("D"); var undirectedType = RelationshipType.of("U"); - var directed = RelationshipSchema.empty() + var directed = MutableRelationshipSchema.empty() .addProperty(directedType, Direction.DIRECTED, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT); - var undirected = RelationshipSchema.empty() + var undirected = MutableRelationshipSchema.empty() .addProperty(undirectedType, Direction.UNDIRECTED, "flob", ValueType.DOUBLE, PropertyState.PERSISTENT); var mixed = directed.union(undirected); @@ -164,13 +160,13 @@ void testUnion(Direction direction1, Direction direction2, Boolean isUndirectedE var type1 = RelationshipType.of("Foo"); var type2 = RelationshipType.of("Bar"); - var relationshipSchema1 = RelationshipSchema.empty() + var relationshipSchema1 = MutableRelationshipSchema.empty() .addProperty(type1, direction1, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT); - var relationshipSchema2 = RelationshipSchema.empty() + var relationshipSchema2 = MutableRelationshipSchema.empty() .addProperty(type2, direction2, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT); - var expected = RelationshipSchema.empty() + var expected = MutableRelationshipSchema.empty() .addProperty(type1, direction1, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(type2, direction2, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT); @@ -184,11 +180,11 @@ void testUnion(Direction direction1, Direction direction2, Boolean isUndirectedE @Test void unionOnSameTypesFailsOnDirectionMismatch() { - var schema1 = RelationshipSchema.empty() + var schema1 = MutableRelationshipSchema.empty() .addRelationshipType(RelationshipType.of("X"), Direction.DIRECTED) .addRelationshipType(RelationshipType.of("Y"), Direction.UNDIRECTED); - var schema2 = RelationshipSchema.empty() + var schema2 = MutableRelationshipSchema.empty() .addRelationshipType(RelationshipType.of("X"), Direction.UNDIRECTED) .addRelationshipType(RelationshipType.of("Y"), Direction.UNDIRECTED); @@ -199,7 +195,7 @@ void unionOnSameTypesFailsOnDirectionMismatch() { @Test void unionOnSameTypeSamePropertyDifferentValueTypeFails() { - var schema1 = RelationshipSchema + var schema1 = MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("X"), Direction.DIRECTED, "x", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(RelationshipType.of("Y"), @@ -209,7 +205,7 @@ void unionOnSameTypeSamePropertyDifferentValueTypeFails() { PropertyState.PERSISTENT ); - var schema2 = RelationshipSchema + var schema2 = MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("X"), Direction.DIRECTED, "x", ValueType.LONG, PropertyState.PERSISTENT) .addProperty(RelationshipType.of("Y"), @@ -229,13 +225,15 @@ void unionOnSameTypeSamePropertyDifferentValueTypeFails() { static Stream schemaAndHasProperties() { return Stream.of( - Arguments.of(RelationshipSchema.empty().addRelationshipType(RelationshipType.of("A"), Direction.DIRECTED), + Arguments.of( + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("A"), Direction.DIRECTED), false ), - Arguments.of(RelationshipSchema.empty().addRelationshipType(RelationshipType.of("A"), Direction.UNDIRECTED), + Arguments.of( + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("A"), Direction.UNDIRECTED), false ), - Arguments.of(RelationshipSchema + Arguments.of(MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("A"), Direction.DIRECTED, @@ -243,7 +241,7 @@ static Stream schemaAndHasProperties() { ValueType.LONG, PropertyState.PERSISTENT ), true), - Arguments.of(RelationshipSchema + Arguments.of(MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("A"), Direction.UNDIRECTED, @@ -256,13 +254,13 @@ static Stream schemaAndHasProperties() { @ParameterizedTest @MethodSource("schemaAndHasProperties") - void shouldKnowIfPropertiesArePresent(RelationshipSchema relationshipSchema, boolean hasProperties) { + void shouldKnowIfPropertiesArePresent(MutableRelationshipSchema relationshipSchema, boolean hasProperties) { assertThat(relationshipSchema.hasProperties()).isEqualTo(hasProperties); } @Test void testBuildRelTypes() { - var schema = RelationshipSchema.empty(); + var schema = MutableRelationshipSchema.empty(); schema.addRelationshipType(RelationshipType.of("X"), Direction.UNDIRECTED); schema.addRelationshipType(RelationshipType.of("Y"), Direction.DIRECTED); @@ -276,7 +274,7 @@ void testBuildRelTypes() { @Test void testBuildProperties() { - RelationshipSchema relationshipSchema = RelationshipSchema.empty() + MutableRelationshipSchema relationshipSchema = MutableRelationshipSchema.empty() .addProperty(RelationshipType.of("X"), Direction.UNDIRECTED, "x", ValueType.DOUBLE, PropertyState.PERSISTENT ); @@ -316,7 +314,7 @@ void testBuildProperties() { @Test void shouldCreateDeepCopiesWhenFiltering() { var relType = RelationshipType.of("A"); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipSchema.getOrCreateRelationshipType(relType, Direction.DIRECTED).addProperty("prop", ValueType.LONG, PropertyState.PERSISTENT ); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporterBaseConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporterBaseConfig.java index 6c83af8280a..3025e5d2a46 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporterBaseConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporterBaseConfig.java @@ -51,8 +51,8 @@ default int batchSize() { @Value.Default @Value.Parameter(false) - @Configuration.ConvertWith(method = "org.neo4j.gds.AbstractPropertyMappings#fromObject") - @Configuration.ToMapValue("org.neo4j.gds.AbstractPropertyMappings#toObject") + @Configuration.ConvertWith(method = "org.neo4j.gds.PropertyMappings#fromObject") + @Configuration.ToMapValue("org.neo4j.gds.PropertyMappings#toObject") default org.neo4j.gds.PropertyMappings additionalNodeProperties() { return org.neo4j.gds.PropertyMappings.of(); } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java index 0384079262b..7a74e29dbc8 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java @@ -73,18 +73,18 @@ enum IdMode implements Supplier { MAPPING(IdType.INTEGER, new Groups()) { @Override public InputEntityIdVisitor.Long get() { - return Neo4jProxy.inputEntityLongIdVisitor(IdType.INTEGER); + return Neo4jProxy.inputEntityLongIdVisitor(IdType.INTEGER, readableGroups); } }, ACTUAL(IdType.ACTUAL, Groups.EMPTY) { @Override public InputEntityIdVisitor.Long get() { - return Neo4jProxy.inputEntityLongIdVisitor(IdType.ACTUAL); + return Neo4jProxy.inputEntityLongIdVisitor(IdType.ACTUAL, readableGroups); } }; private final IdType idType; - private final ReadableGroups readableGroups; + final ReadableGroups readableGroups; IdMode(IdType idType, ReadableGroups readableGroups) { this.idType = idType; diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreRelationshipVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreRelationshipVisitor.java index 6dfab81155b..dae414f417c 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreRelationshipVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreRelationshipVisitor.java @@ -62,16 +62,19 @@ protected void exportElement() { // TODO: this logic should move to the RelationshipsBuilder var relationshipsBuilder = relationshipFromVisitorBuilders.computeIfAbsent( relationshipType(), - (relationshipType) -> { + (relationshipTypeString) -> { var propertyConfigs = getPropertySchema() .stream() .map(schema -> GraphFactory.PropertyConfig.of(schema.key(), schema.aggregation(), schema.defaultValue())) .collect(Collectors.toList()); + RelationshipType relationshipType = RelationshipType.of(relationshipTypeString); + var relBuilder = relationshipBuilderSupplier.get() + .relationshipType(relationshipType) .propertyConfigs(propertyConfigs) - .indexInverse(inverseIndexedRelationshipTypes.contains(RelationshipType.of(relationshipType))) + .indexInverse(inverseIndexedRelationshipTypes.contains(relationshipType)) .build(); - relationshipBuilders.put(relationshipType, relBuilder); + relationshipBuilders.put(relationshipTypeString, relBuilder); return RelationshipBuilderFromVisitor.of( propertyConfigs.size(), relBuilder, diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfig.java index 7282a6d3b38..da7b0912b6d 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfig.java @@ -107,7 +107,7 @@ default org.neo4j.internal.batchimport.Configuration toBatchImporterConfig() { exportConfig.writeConcurrency(), exportConfig.pageCacheMemory(), exportConfig.highIO(), - IndexConfig.DEFAULT.withLabelIndex() + IndexConfig.DEFAULT.withLabelIndex().withRelationshipTypeIndex() ); } } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java index 728a238b00e..56fbfccccf8 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java @@ -114,8 +114,8 @@ public void start(StageExecution execution) { this.stageProgressCurrent.set(0); var neoStores = this.dependencyResolver.resolveDependency(BatchingNeoStores.class); - var relationshipRecordIdCount = neoStores.getRelationshipStore().getHighId(); - var groupCount = neoStores.getTemporaryRelationshipGroupStore().getHighId(); + var relationshipRecordIdCount = Neo4jProxy.getHighId(neoStores.getRelationshipStore()); + var groupCount = Neo4jProxy.getHighId(neoStores.getTemporaryRelationshipGroupStore()); switch (execution.getStageName()) { case NodeDegreeCountStage.NAME: diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java index 1b9ae3f02a1..5122e3a5e50 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java @@ -19,9 +19,9 @@ */ package org.neo4j.gds.core.io.file; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.compat.CompatInput; import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.internal.batchimport.InputIterable; @@ -32,8 +32,8 @@ public interface FileInput extends CompatInput { InputIterable graphProperties(); String userName(); GraphInfo graphInfo(); - NodeSchema nodeSchema(); - RelationshipSchema relationshipSchema(); + MutableNodeSchema nodeSchema(); + MutableRelationshipSchema relationshipSchema(); Map graphPropertySchema(); Capabilities capabilities(); } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 8955203911d..1bda4e211c0 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -26,16 +26,15 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.RelationshipPropertyStore; import org.neo4j.gds.api.Topology; -import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.ImmutableMutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.io.GraphStoreGraphPropertyVisitor; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; -import org.neo4j.gds.core.loading.CSRGraphStoreUtil; import org.neo4j.gds.core.loading.GraphStoreBuilder; -import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; +import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodesBuilder; @@ -52,7 +51,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -65,7 +63,7 @@ public abstract class FileToGraphStoreImporter { private final Path importPath; private final int concurrency; - private final ImmutableGraphSchema.Builder graphSchemaBuilder; + private final ImmutableMutableGraphSchema.Builder graphSchemaBuilder; private final GraphStoreBuilder graphStoreBuilder; private final Log log; private final TaskRegistryFactory taskRegistryFactory; @@ -83,7 +81,7 @@ protected FileToGraphStoreImporter( this.graphPropertyVisitorBuilder = new GraphStoreGraphPropertyVisitor.Builder(); this.concurrency = concurrency; this.importPath = importPath; - this.graphSchemaBuilder = ImmutableGraphSchema.builder(); + this.graphSchemaBuilder = ImmutableMutableGraphSchema.builder(); this.graphStoreBuilder = new GraphStoreBuilder() .concurrency(concurrency) // TODO: we need to export and import this flag: https://trello.com/c/2cEMPZ9L @@ -99,13 +97,19 @@ protected FileToGraphStoreImporter( public UserGraphStore run() { var fileInput = fileInput(importPath); this.progressTracker = createProgressTracker(fileInput); - progressTracker.beginSubTask(); + try { + progressTracker.beginSubTask(); importGraphStore(fileInput); graphStoreBuilder.schema(graphSchemaBuilder.build()); - return ImmutableUserGraphStore.of(fileInput.userName(), graphStoreBuilder.build()); - } finally { + var userGraphStore = ImmutableUserGraphStore.of(fileInput.userName(), graphStoreBuilder.build()); progressTracker.endSubTask(); + + return userGraphStore; + } catch (Exception e) { + progressTracker.endSubTaskWithFailure(); + + throw e; } } @@ -138,13 +142,13 @@ private void importGraphStore(FileInput fileInput) { graphStoreBuilder.capabilities(fileInput.capabilities()); var nodes = importNodes(fileInput); - importRelationships(fileInput, nodes); + importRelationships(fileInput, nodes.idMap()); importGraphProperties(fileInput); } - private IdMap importNodes(FileInput fileInput) { + private Nodes importNodes(FileInput fileInput) { progressTracker.beginSubTask(); - NodeSchema nodeSchema = fileInput.nodeSchema(); + MutableNodeSchema nodeSchema = fileInput.nodeSchema(); graphSchemaBuilder.nodeSchema(nodeSchema); NodesBuilder nodesBuilder = GraphFactory.initNodesBuilder(nodeSchema) @@ -165,19 +169,12 @@ private IdMap importNodes(FileInput fileInput) { ParallelUtil.run(tasks, Pools.DEFAULT); var nodes = nodesBuilder.build(); - var nodeImportResultBuilder = ImmutableNodes.builder().idMap(nodes.idMap()); - var schemaProperties = nodeSchema.unionProperties(); - CSRGraphStoreUtil.extractNodeProperties( - nodeImportResultBuilder, - schemaProperties::get, - nodes.properties().propertyValues() - ); + this.graphStoreBuilder.nodes(nodes); - graphStoreBuilder.nodes(nodeImportResultBuilder.build()); + this.progressTracker.endSubTask(); - progressTracker.endSubTask(); - return nodes.idMap(); + return nodes; } private void importRelationships(FileInput fileInput, IdMap nodes) { @@ -202,12 +199,7 @@ private void importRelationships(FileInput fileInput, IdMap nodes) { ParallelUtil.run(tasks, Pools.DEFAULT); - var relationships = relationshipTopologyAndProperties(relationshipBuildersByType); - var relationshipImportResult = RelationshipImportResult.of( - relationships.topologies(), - relationships.properties(), - relationshipSchema.directions() - ); + var relationshipImportResult = relationshipImportResult(relationshipBuildersByType); graphStoreBuilder.relationshipImportResult(relationshipImportResult); @@ -246,30 +238,15 @@ private void importGraphProperties(FileInput fileInput) { } } - public static RelationshipTopologyAndProperties relationshipTopologyAndProperties(Map relationshipBuildersByType) { - var propertyStores = new HashMap(); - var relationshipTypeTopologyMap = relationshipTypeToTopologyMapping( - relationshipBuildersByType, - propertyStores - ); - - var importedRelationships = relationshipTypeTopologyMap.values().stream().mapToLong(Topology::elementCount).sum(); - return ImmutableRelationshipTopologyAndProperties.of(relationshipTypeTopologyMap, propertyStores, importedRelationships); - } + public static RelationshipImportResult relationshipImportResult(Map relationshipBuildersByType) { + var relationshipsByType = relationshipBuildersByType.entrySet() + .stream() + .collect(Collectors.toMap( + e -> RelationshipType.of(e.getKey()), + e -> e.getValue().build() + )); - private static Map relationshipTypeToTopologyMapping( - Map relationshipBuildersByType, - Map propertyStores - ) { - return relationshipBuildersByType.entrySet().stream().map(entry -> { - var relationshipType = RelationshipType.of(entry.getKey()); - var relationships = entry.getValue().build(); - - if (relationships.properties().isPresent()) { - propertyStores.put(relationshipType, relationships.properties().get()); - } - return Map.entry(relationshipType, relationships.topology()); - }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return RelationshipImportResult.builder().importResults(relationshipsByType).build(); } @ValueClass diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java index 0a7b781c7bf..506e000df11 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java @@ -93,7 +93,7 @@ static class LongStreamBuilderGraphPropertyValues implements LongGraphPropertyVa } @Override - public long size() { + public long valueCount() { return -1; } @@ -112,7 +112,7 @@ static class DoubleStreamBuilderGraphPropertyValues implements DoubleGraphProper } @Override - public long size() { + public long valueCount() { return -1; } @@ -132,7 +132,7 @@ static class FloatArrayStreamBuilderGraphPropertyValues implements FloatArrayGra } @Override - public long size() { + public long valueCount() { return -1; } @@ -151,7 +151,7 @@ static class DoubleArrayStreamBuilderGraphPropertyValues implements DoubleArrayG } @Override - public long size() { + public long valueCount() { return -1; } @@ -170,7 +170,7 @@ static class LongArrayStreamBuilderGraphPropertyValues implements LongArrayGraph } @Override - public long size() { + public long valueCount() { return -1; } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitor.java index bf125ad3290..e8b12ed8c29 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitor.java @@ -19,15 +19,15 @@ */ package org.neo4j.gds.core.io.schema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.PropertySchema; public class NodeSchemaBuilderVisitor extends NodeSchemaVisitor { - private final NodeSchema nodeSchema; + private final MutableNodeSchema nodeSchema; public NodeSchemaBuilderVisitor() { - nodeSchema = NodeSchema.empty(); + nodeSchema = MutableNodeSchema.empty(); } @Override @@ -42,7 +42,7 @@ protected void export() { } } - public NodeSchema schema() { + public MutableNodeSchema schema() { return nodeSchema; } } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/schema/RelationshipSchemaBuilderVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/schema/RelationshipSchemaBuilderVisitor.java index 0abafde4a61..f4fc164a0e3 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/schema/RelationshipSchemaBuilderVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/schema/RelationshipSchemaBuilderVisitor.java @@ -19,15 +19,15 @@ */ package org.neo4j.gds.core.io.schema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; public class RelationshipSchemaBuilderVisitor extends RelationshipSchemaVisitor { - private final RelationshipSchema schema; + private final MutableRelationshipSchema schema; public RelationshipSchemaBuilderVisitor() { - this.schema = RelationshipSchema.empty(); + this.schema = MutableRelationshipSchema.empty(); } @Override @@ -41,7 +41,7 @@ protected void export() { } } - public RelationshipSchema schema() { + public MutableRelationshipSchema schema() { return schema; } } diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java index a660836167b..fe2ee835695 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java @@ -40,9 +40,10 @@ void testToBatchImporterConfig() { var pbiConfig = config.toBatchImporterConfig(); assertThat(pbiConfig.batchSize()).isEqualTo(1337); - assertThat(pbiConfig.pageCacheMemory()).isEqualTo(100_000L); assertThat(pbiConfig.highIO()).isTrue(); assertThat(Neo4jProxy.writeConcurrency(pbiConfig)).isEqualTo(42); + assertThat(pbiConfig.indexConfig().createLabelIndex()).isTrue(); + assertThat(pbiConfig.indexConfig().createRelationshipIndex()).isTrue(); } } diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreNodeVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreNodeVisitorTest.java index f4bff87c51f..0461ef3fe07 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreNodeVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreNodeVisitorTest.java @@ -25,9 +25,9 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -56,7 +56,7 @@ class GraphStoreNodeVisitorTest { @Test void shouldAddNodesToNodesBuilder() { - NodeSchema nodeSchema = NodeSchema.from(graphStore.schema().nodeSchema()); + MutableNodeSchema nodeSchema = MutableNodeSchema.from(graphStore.schema().nodeSchema()); NodesBuilder nodesBuilder = GraphFactory.initNodesBuilder(nodeSchema) .concurrency(1) .maxOriginalId(graphStore.nodeCount()) @@ -79,10 +79,10 @@ void shouldAddNodesToNodesBuilder() { var idMap = nodes.idMap(); var nodeProperties = nodes.properties().propertyValues(); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipSchema.getOrCreateRelationshipType(RelationshipType.ALL_RELATIONSHIPS, Direction.UNDIRECTED); - var graphSchema = GraphSchema.of( + var graphSchema = MutableGraphSchema.of( nodeSchema, relationshipSchema, Map.of() diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java index e6ba8e67ffe..b3aa14ca57f 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java @@ -24,12 +24,12 @@ import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.api.RelationshipPropertyStore; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; import org.neo4j.gds.core.loading.GraphStoreBuilder; +import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; -import org.neo4j.gds.core.loading.Nodes; -import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; import org.neo4j.gds.core.loading.construction.RelationshipsBuilderBuilder; @@ -148,19 +148,26 @@ private Graph createGraph( Map relationshipBuildersByType, long expectedImportedRelationshipsCount ) { - var actualRelationships = FileToGraphStoreImporter.relationshipTopologyAndProperties(relationshipBuildersByType); - assertThat(actualRelationships.importedRelationships()).isEqualTo(expectedImportedRelationshipsCount); + var actualRelationships = FileToGraphStoreImporter.relationshipImportResult(relationshipBuildersByType); + var actualRelationshipCount = actualRelationships + .importResults() + .values() + .stream() + .mapToLong(r -> r.topology().elementCount()) + .sum(); + + assertThat(actualRelationshipCount).isEqualTo(expectedImportedRelationshipsCount); + + var nodes = ImmutableNodes.builder() + .idMap(expectedGraph) + .schema(MutableNodeSchema.from(expectedGraph.schema().nodeSchema())) + .build(); - Map propertyStores = actualRelationships.properties(); return new GraphStoreBuilder() - .schema(expectedGraph.schema()) + .schema(MutableGraphSchema.from(expectedGraph.schema())) .capabilities(ImmutableStaticCapabilities.of(true)) - .nodes(Nodes.of(expectedGraph)) - .relationshipImportResult(RelationshipImportResult.of( - actualRelationships.topologies(), - propertyStores, - expectedGraph.schema().relationshipSchema().directions() - )) + .nodes(nodes) + .relationshipImportResult(actualRelationships) .databaseId(DatabaseId.random()) .concurrency(1) .build() diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitorTest.java index 151064d9b78..66de46b4261 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitorTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchemaEntry; +import org.neo4j.gds.api.schema.MutableNodeSchemaEntry; import org.neo4j.gds.api.schema.PropertySchema; import java.util.Map; @@ -62,7 +62,7 @@ void shouldBuildNodeSchema() { var labelAEntry = builtSchema.get(NodeLabel.of("A")); assertThat(labelAEntry) .isEqualTo( - new NodeSchemaEntry( + new MutableNodeSchemaEntry( NodeLabel.of("A"), Map.of( "prop1", @@ -79,7 +79,7 @@ void shouldBuildNodeSchema() { var labelBEntry = builtSchema.get(NodeLabel.of("B")); assertThat(labelBEntry) .isEqualTo( - new NodeSchemaEntry( + new MutableNodeSchemaEntry( NodeLabel.of("B"), Map.of( "prop2", diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/NodeFileHeader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/NodeFileHeader.java index edb4f21e027..198dc3636af 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/NodeFileHeader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/NodeFileHeader.java @@ -21,7 +21,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.core.io.file.csv.CsvNodeVisitor; import org.neo4j.gds.utils.StringFormatting; @@ -33,11 +33,11 @@ import java.util.stream.Stream; @ValueClass -public interface NodeFileHeader extends FileHeader { +public interface NodeFileHeader extends FileHeader { String[] nodeLabels(); @Override - default Map schemaForIdentifier(NodeSchema schema) { + default Map schemaForIdentifier(MutableNodeSchema schema) { var labelStream = Arrays.stream(nodeLabels()).map(NodeLabel::of); if (nodeLabels().length == 0) { labelStream = Stream.of(NodeLabel.ALL_NODES); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/RelationshipFileHeader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/RelationshipFileHeader.java index bb692eee7e8..3442ab2e3c9 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/RelationshipFileHeader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/RelationshipFileHeader.java @@ -21,8 +21,8 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.io.file.csv.CsvRelationshipVisitor; import org.neo4j.gds.utils.StringFormatting; @@ -30,11 +30,11 @@ import java.util.Set; @ValueClass -public interface RelationshipFileHeader extends FileHeader { +public interface RelationshipFileHeader extends FileHeader { String relationshipType(); @Override - default Map schemaForIdentifier(RelationshipSchema schema) { + default Map schemaForIdentifier(MutableRelationshipSchema schema) { return schema.filter(Set.of(RelationshipType.of(relationshipType()))).unionProperties(); } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java index 4b9d67659c1..4a4a776d3f1 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java @@ -25,10 +25,10 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.apache.commons.lang3.tuple.Pair; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.compat.CompatPropertySizeCalculator; import org.neo4j.gds.core.io.GraphStoreInput; import org.neo4j.gds.core.io.file.FileHeader; @@ -79,8 +79,8 @@ final class CsvFileInput implements FileInput { private final Path importPath; private final String userName; private final GraphInfo graphInfo; - private final NodeSchema nodeSchema; - private final RelationshipSchema relationshipSchema; + private final MutableNodeSchema nodeSchema; + private final MutableRelationshipSchema relationshipSchema; private final Map graphPropertySchema; private final Capabilities capabilities; @@ -153,12 +153,12 @@ public GraphInfo graphInfo() { } @Override - public NodeSchema nodeSchema() { + public MutableNodeSchema nodeSchema() { return nodeSchema; } @Override - public RelationshipSchema relationshipSchema() { + public MutableRelationshipSchema relationshipSchema() { return relationshipSchema; } @@ -206,11 +206,11 @@ public void close() { } } - static class NodeImporter extends FileImporter { + static class NodeImporter extends FileImporter { NodeImporter( Map> headerToDataFilesMapping, - NodeSchema nodeSchema + MutableNodeSchema nodeSchema ) { super(headerToDataFilesMapping, nodeSchema); } @@ -221,11 +221,11 @@ public InputChunk newChunk() { } } - static class RelationshipImporter extends FileImporter { + static class RelationshipImporter extends FileImporter { RelationshipImporter( Map> headerToDataFilesMapping, - RelationshipSchema relationshipSchema + MutableRelationshipSchema relationshipSchema ) { super(headerToDataFilesMapping, relationshipSchema); } @@ -301,9 +301,9 @@ public long lastProgress() { } } - static class NodeLineChunk extends LineChunk { + static class NodeLineChunk extends LineChunk { - NodeLineChunk(NodeSchema nodeSchema) { + NodeLineChunk(MutableNodeSchema nodeSchema) { super(nodeSchema); } @@ -318,9 +318,9 @@ void visitLine(String[] lineArray, NodeFileHeader header, InputEntityVisitor vis } } - static class RelationshipLineChunk extends LineChunk { + static class RelationshipLineChunk extends LineChunk { - RelationshipLineChunk(RelationshipSchema relationshipSchema) { + RelationshipLineChunk(MutableRelationshipSchema relationshipSchema) { super(relationshipSchema); } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java index 3b50879d643..e606f114b9f 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.io.NeoNodeProperties; import org.neo4j.gds.core.io.file.GraphStoreToFileExporter; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfig; @@ -59,7 +59,7 @@ public static GraphStoreToFileExporter create( var nodeSchema = graphStore.schema().nodeSchema(); var relationshipSchema = graphStore.schema().relationshipSchema(); - var neoNodeSchema = NodeSchema.empty(); + var neoNodeSchema = MutableNodeSchema.empty(); // Add additional properties to each label present in the graph store. neoNodeProperties.ifPresent(additionalProps -> additionalProps diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java index c8f891680da..18fbfee3dfc 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java @@ -29,7 +29,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.io.schema.NodeSchemaBuilderVisitor; import java.io.BufferedReader; @@ -51,7 +51,7 @@ public class NodeSchemaLoader { objectReader = csvMapper.readerFor(SchemaLine.class).with(schema); } - NodeSchema load() { + MutableNodeSchema load() { NodeSchemaBuilderVisitor schemaBuilder = new NodeSchemaBuilderVisitor(); try(var reader = new BufferedReader(new FileReader(nodeSchemaPath.toFile(), StandardCharsets.UTF_8))) { diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java index 07973f7c3c3..d2802af58aa 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java @@ -30,7 +30,7 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.io.schema.RelationshipSchemaBuilderVisitor; @@ -53,7 +53,7 @@ public class RelationshipSchemaLoader { objectReader = csvMapper.readerFor(SchemaLine.class).with(schema); } - RelationshipSchema load() { + MutableRelationshipSchema load() { var schemaBuilder = new RelationshipSchemaBuilderVisitor(); try (var reader = new BufferedReader(new FileReader(relationshipSchemaPath.toFile(), StandardCharsets.UTF_8))) { diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitorTest.java index c0c37d6267a..b6b690bf79e 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitorTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import java.util.Collections; import java.util.List; @@ -34,7 +34,7 @@ class CsvNodeVisitorTest extends CsvVisitorTest { @Test void visitNodesWithoutLabelsAndProperties() { - var nodeVisitor = new CsvNodeVisitor(tempDir, NodeSchema.empty()); + var nodeVisitor = new CsvNodeVisitor(tempDir, MutableNodeSchema.empty()); nodeVisitor.id(0L); nodeVisitor.endOfEntity(); @@ -55,7 +55,7 @@ void visitNodesWithoutLabelsAndProperties() { @Test void visitNodesWithLabels() { - var nodeVisitor = new CsvNodeVisitor(tempDir, NodeSchema.empty()); + var nodeVisitor = new CsvNodeVisitor(tempDir, MutableNodeSchema.empty()); nodeVisitor.id(0L); nodeVisitor.labels(new String[]{"Foo", "Bar"}); @@ -92,7 +92,7 @@ void visitNodesWithLabels() { @Test void visitNodesWithProperties() { - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema.getOrCreateLabel(NodeLabel.ALL_NODES) .addProperty("foo", ValueType.DOUBLE) .addProperty("bar", ValueType.DOUBLE); @@ -131,7 +131,7 @@ void visitNodesWithLabelsAndProperties() { var bLabel = NodeLabel.of("B"); var cLabel = NodeLabel.of("C"); - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema.getOrCreateLabel(aLabel) .addProperty("foo", ValueType.LONG) .addProperty("bar", ValueType.LONG); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvRelationshipVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvRelationshipVisitorTest.java index 0544b7deacb..68d243e2054 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvRelationshipVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvRelationshipVisitorTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import java.util.Collections; import java.util.List; @@ -37,7 +37,7 @@ class CsvRelationshipVisitorTest extends CsvVisitorTest { @Test void visitRelationshipsWithTypes() { - var relationshipVisitor = new CsvRelationshipVisitor(tempDir, RelationshipSchema.empty()); + var relationshipVisitor = new CsvRelationshipVisitor(tempDir, MutableRelationshipSchema.empty()); relationshipVisitor.startId(0L); relationshipVisitor.endId(1L); @@ -80,7 +80,7 @@ void visitRelationshipsWithTypesAndProperties() { var aType = RelationshipType.of("A"); var bType = RelationshipType.of("B"); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipSchema.getOrCreateRelationshipType(aType, Direction.DIRECTED) .addProperty("foo", ValueType.LONG, PropertyState.PERSISTENT) .addProperty("bar", ValueType.LONG, PropertyState.PERSISTENT); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index 65d429e88eb..d85d7f397f2 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -168,7 +168,7 @@ public Stream doubleArrayValues() { } @Override - public long size() { + public long valueCount() { return 1337; } }); @@ -182,7 +182,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valueCount() { return 10_000; } }); @@ -196,7 +196,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valueCount() { return 10_000; } }); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java index e00324627dd..54a4ed41e59 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java @@ -235,13 +235,13 @@ void shouldExportGraphProperties() { var graphPropertyValues = new LongGraphPropertyValues() { @Override - public long size() { + public long valueCount() { return 3; } @Override public LongStream longValues() { - return LongStream.range(0, size()); + return LongStream.range(0, valueCount()); } }; @@ -337,14 +337,15 @@ void exportGraphPropertiesMultithreaded() throws IOException { .build(); var graphPropertyValues = new LongGraphPropertyValues() { + @Override - public long size() { + public long valueCount() { return 1_000_000; } @Override public LongStream longValues() { - return LongStream.range(0, size()); + return LongStream.range(0, valueCount()); } }; @@ -386,7 +387,7 @@ public LongStream longValues() { .sorted() .toArray(); - assertArrayEquals(LongStream.range(0, graphPropertyValues.size()).toArray(), exportedValues); + assertArrayEquals(LongStream.range(0, graphPropertyValues.valueCount()).toArray(), exportedValues); } @Test @@ -401,11 +402,11 @@ void exportSchemaAndDatabaseId() { graphStore.addGraphProperty("graphProp", new LongGraphPropertyValues() { @Override public LongStream longValues() { - return LongStream.range(0, size()); + return LongStream.range(0, valueCount()); } @Override - public long size() { + public long valueCount() { return 3; } }); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java index c8365e3b07d..2367508e6c9 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchemaEntry; +import org.neo4j.gds.api.schema.MutableNodeSchemaEntry; import org.neo4j.gds.api.schema.PropertySchema; import java.io.IOException; @@ -61,7 +61,7 @@ void shouldLoadNodeSchemaCorrectly() throws IOException { var labelAProperties = nodeSchema.get(NodeLabel.of("A")); assertThat(labelAProperties) - .isEqualTo(new NodeSchemaEntry( + .isEqualTo(new MutableNodeSchemaEntry( NodeLabel.of("A"), Map.of( "prop1", @@ -76,7 +76,7 @@ void shouldLoadNodeSchemaCorrectly() throws IOException { var labelBProperties = nodeSchema.get(NodeLabel.of("B")); assertThat(labelBProperties) - .isEqualTo(new NodeSchemaEntry( + .isEqualTo(new MutableNodeSchemaEntry( NodeLabel.of("B"), Map.of( "prop2", @@ -126,7 +126,7 @@ void shouldLoadMixedLabels() throws IOException { var labelAProperties = nodeSchema.get(NodeLabel.of("A")); assertThat(labelAProperties) - .isEqualTo(new NodeSchemaEntry( + .isEqualTo(new MutableNodeSchemaEntry( NodeLabel.of("A"), Map.of( "prop1", @@ -141,7 +141,7 @@ void shouldLoadMixedLabels() throws IOException { var labelBProperties = nodeSchema.get(NodeLabel.of("B")); assertThat(labelBProperties) - .isEqualTo(new NodeSchemaEntry(NodeLabel.of("B"), Map.of())); + .isEqualTo(new MutableNodeSchemaEntry(NodeLabel.of("B"), Map.of())); } } diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoaderTest.java index 303daa5dbd1..345c13a0392 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoaderTest.java @@ -27,8 +27,8 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import org.neo4j.gds.core.Aggregation; import java.io.IOException; @@ -59,7 +59,7 @@ void shouldLoadRelationshipSchemaCorrectly() throws IOException { var rel1Properties = loadedRelationshipSchema.get(RelationshipType.of("REL1")); assertThat(rel1Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL1"), Direction.DIRECTED, Map.of( @@ -76,7 +76,7 @@ void shouldLoadRelationshipSchemaCorrectly() throws IOException { var rel2Properties = loadedRelationshipSchema.get(RelationshipType.of("REL2")); assertThat(rel2Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL2"), Direction.UNDIRECTED, Map.of( @@ -125,7 +125,7 @@ void shouldLoadMixedRelationshipSchema() throws IOException { var rel1Properties = loadedRelationshipSchema.get(RelationshipType.of("REL1")); assertThat(rel1Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL1"), Direction.DIRECTED, Map.of( @@ -142,7 +142,7 @@ void shouldLoadMixedRelationshipSchema() throws IOException { var rel3Properties = loadedRelationshipSchema.get(RelationshipType.of("REL3")); assertThat(rel3Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL3"), Direction.UNDIRECTED, Map.of() @@ -166,7 +166,7 @@ void shouldLoadRelSchemaWithoutOrientation() throws IOException { var rel1Properties = loadedRelationshipSchema.get(RelationshipType.of("REL1")); assertThat(rel1Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL1"), Direction.DIRECTED, Map.of( @@ -183,7 +183,7 @@ void shouldLoadRelSchemaWithoutOrientation() throws IOException { var rel3Properties = loadedRelationshipSchema.get(RelationshipType.of("REL3")); assertThat(rel3Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL3"), Direction.DIRECTED, Map.of() diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/metrics/classification/GlobalAccuracy.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/metrics/classification/GlobalAccuracy.java index a97c45b4950..6c1d8e5ff26 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/metrics/classification/GlobalAccuracy.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/metrics/classification/GlobalAccuracy.java @@ -24,6 +24,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Comparator; +import java.util.Objects; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -69,7 +70,7 @@ public double compute(HugeIntArray targets, HugeIntArray predictions) { @Override public int hashCode() { - return super.hashCode(); + return Objects.hash(NAME); } @Override diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/NegativeSampler.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/NegativeSampler.java index 0f222872440..40a0f92f6cd 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/NegativeSampler.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/NegativeSampler.java @@ -27,6 +27,7 @@ import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; import java.util.Collection; +import java.util.List; import java.util.Optional; public interface NegativeSampler { @@ -36,6 +37,7 @@ public interface NegativeSampler { static NegativeSampler of( GraphStore graphStore, Graph graph, + Collection sourceAndTargetNodeLabels, Optional negativeRelationshipType, double negativeSamplingRatio, long testPositiveCount, @@ -47,7 +49,11 @@ static NegativeSampler of( Optional randomSeed ) { if (negativeRelationshipType.isPresent()) { - Graph negativeExampleGraph = graphStore.getGraph(RelationshipType.of(negativeRelationshipType.orElseThrow())); + Graph negativeExampleGraph = graphStore.getGraph( + sourceAndTargetNodeLabels, + List.of(RelationshipType.of(negativeRelationshipType.orElseThrow())), + Optional.empty() + ); double testTrainFraction = testPositiveCount / (double) (testPositiveCount + trainPositiveCount); return new UserInputNegativeSampler( diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/UserInputNegativeSampler.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/UserInputNegativeSampler.java index b19af6fc804..7a3db98a57f 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/UserInputNegativeSampler.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/UserInputNegativeSampler.java @@ -64,13 +64,16 @@ public void produceNegativeSamples( negativeExampleGraph.forEachNode(nodeId -> { negativeExampleGraph.forEachRelationship(nodeId, (s, t) -> { + // as we are adding the relationships to the GraphStore we need to operate over the rootNodeIds + long rootS = negativeExampleGraph.toRootNodeId(s); + long rootT = negativeExampleGraph.toRootNodeId(t); if (s < t) { if (sample(testRelationshipsToAdd.doubleValue()/(testRelationshipsToAdd.doubleValue() + trainRelationshipsToAdd.doubleValue()))) { testRelationshipsToAdd.decrement(); - testSetBuilder.add(s, t, NEGATIVE); + testSetBuilder.addFromInternal(rootS, rootT, NEGATIVE); } else { trainRelationshipsToAdd.decrement(); - trainSetBuilder.add(s, t, NEGATIVE); + trainSetBuilder.addFromInternal(rootS, rootT, NEGATIVE); } } return true; diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java index e7c00c7ab63..3c86dff2ba4 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.predicates.LongLongPredicate; import org.apache.commons.lang3.mutable.MutableLong; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; @@ -35,11 +36,22 @@ public class DirectedEdgeSplitter extends EdgeSplitter { public DirectedEdgeSplitter( Optional maybeSeed, + IdMap rootNodes, IdMap sourceLabels, IdMap targetLabels, + RelationshipType selectedRelationshipType, + RelationshipType remainingRelationshipType, int concurrency ) { - super(maybeSeed, sourceLabels, targetLabels, concurrency); + super( + maybeSeed, + rootNodes, + sourceLabels, + targetLabels, + selectedRelationshipType, + remainingRelationshipType, + concurrency + ); } @Override diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java index bb45719edd1..db52570ae92 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java @@ -22,6 +22,7 @@ import com.carrotsearch.hppc.predicates.LongLongPredicate; import com.carrotsearch.hppc.predicates.LongPredicate; import org.apache.commons.lang3.mutable.MutableLong; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.Graph; @@ -41,12 +42,27 @@ public abstract class EdgeSplitter { public static final double POSITIVE = 1D; public static final String RELATIONSHIP_PROPERTY = "label"; private final Random rng; + private final RelationshipType selectedRelationshipType; + private final RelationshipType remainingRelationshipType; protected final IdMap sourceNodes; protected final IdMap targetNodes; + protected final IdMap rootNodes; + protected int concurrency; - EdgeSplitter(Optional maybeSeed, IdMap sourceNodes, IdMap targetNodes, int concurrency) { + EdgeSplitter( + Optional maybeSeed, + IdMap rootNodes, + IdMap sourceNodes, + IdMap targetNodes, + RelationshipType selectedRelationshipType, + RelationshipType remainingRelationshipType, + int concurrency + ) { + this.rootNodes = rootNodes; + this.selectedRelationshipType = selectedRelationshipType; + this.remainingRelationshipType = remainingRelationshipType; this.rng = new Random(); maybeSeed.ifPresent(rng::setSeed); @@ -65,8 +81,10 @@ public SplitResult splitPositiveExamples( LongLongPredicate isValidNodePair = (s, t) -> isValidSourceNode.apply(s) && isValidTargetNode.apply(t); RelationshipsBuilder selectedRelsBuilder = newRelationshipsBuilder( - graph, - Direction.DIRECTED, Optional.of(EdgeSplitter.RELATIONSHIP_PROPERTY) + rootNodes, + selectedRelationshipType, + Direction.DIRECTED, + Optional.of(EdgeSplitter.RELATIONSHIP_PROPERTY) ); Direction remainingRelDirection = graph.schema().direction(); @@ -74,7 +92,7 @@ public SplitResult splitPositiveExamples( RelationshipsBuilder remainingRelsBuilder; RelationshipWithPropertyConsumer remainingRelsConsumer; - remainingRelsBuilder = newRelationshipsBuilder(graph, remainingRelDirection, remainingRelPropertyKey); + remainingRelsBuilder = newRelationshipsBuilder(rootNodes, remainingRelationshipType, remainingRelDirection, remainingRelPropertyKey); remainingRelsConsumer = (s, t, w) -> { remainingRelsBuilder.addFromInternal(graph.toRootNodeId(s), graph.toRootNodeId(t), w); return true; @@ -138,13 +156,15 @@ protected long samplesPerNode(long maxSamples, double remainingSamples, long rem } private static RelationshipsBuilder newRelationshipsBuilder( - Graph graph, + IdMap rootNodes, + RelationshipType relationshipType, Direction direction, Optional propertyKey ) { return GraphFactory.initRelationshipsBuilder() + .relationshipType(relationshipType) .aggregation(Aggregation.SINGLE) - .nodes(graph) + .nodes(rootNodes) .orientation(direction.toOrientation()) .addAllPropertyConfigs(propertyKey .map(key -> List.of(GraphFactory.PropertyConfig.of(key, Aggregation.SINGLE, DefaultValue.forDouble()))) diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java index f67d9c8ff02..86563c863c3 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java @@ -40,14 +40,19 @@ public final class SplitRelationships extends Algorithm distinctTargets; + private final SortedSet distinctInternalTargets; public static MemoryEstimation memoryEstimationForNodeSet(int k, double trainFraction) { return memoryEstimation(k, dim -> (long) (dim.nodeCount() * trainFraction)); @@ -79,12 +79,12 @@ public static MemoryEstimation memoryEstimation(int k, ToLongFunction randomSeed, SortedSet distinctTargets) { + public StratifiedKFoldSplitter(int k, ReadOnlyHugeLongArray ids, LongToLongFunction targets, Optional randomSeed, SortedSet distinctInternalTargets) { this.k = k; this.ids = ids; this.targets = targets; this.random = ShuffleUtil.createRandomDataGenerator(randomSeed); - this.distinctTargets = distinctTargets; + this.distinctInternalTargets = distinctInternalTargets; } public List splits() { @@ -97,7 +97,7 @@ public List splits() { allocateArrays(nodeCount, trainSets, testSets); var roundRobinPointer = new MutableInt(); - distinctTargets.forEach(currentClass -> { + distinctInternalTargets.forEach(currentClass -> { for (long offset = 0; offset < ids.size(); offset++) { var id = ids.get(offset); if (targets.applyAsLong(id) == currentClass) { diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitter.java index 778d501d886..aeb04dd5f2f 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitter.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.predicates.LongLongPredicate; import org.apache.commons.lang3.mutable.MutableLong; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; @@ -44,11 +45,16 @@ public class UndirectedEdgeSplitter extends EdgeSplitter { public UndirectedEdgeSplitter( Optional maybeSeed, + IdMap rootNodes, IdMap sourceNodes, IdMap targetNodes, + RelationshipType selectedRelationshipType, + RelationshipType remainingRelationshipType, int concurrency ) { - super(maybeSeed, sourceNodes, targetNodes, concurrency); + super(maybeSeed, + rootNodes, + sourceNodes, targetNodes, selectedRelationshipType, remainingRelationshipType, concurrency); } @Override diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/CrossValidation.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/CrossValidation.java index cb13261ea17..88b4d58e79e 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/CrossValidation.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/CrossValidation.java @@ -85,7 +85,7 @@ public CrossValidation( public void selectModel( ReadOnlyHugeLongArray outerTrainSet, LongToLongFunction targets, - SortedSet distinctTargets, + SortedSet distinctInternalTargets, TrainingStatistics trainingStatistics, Iterator modelCandidates ) { @@ -95,7 +95,7 @@ public void selectModel( outerTrainSet, targets, randomSeed, - distinctTargets + distinctInternalTargets ).splits(); progressTracker.endSubTask("Create validation folds"); diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/TrainingStatistics.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/TrainingStatistics.java index d6265ea9beb..ffbe584e17d 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/TrainingStatistics.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/TrainingStatistics.java @@ -60,6 +60,11 @@ public List getValidationStats(Metric metric) { return modelCandidateStats.stream().map(stats -> stats.validationStats().get(metric)).collect(Collectors.toList()); } + @TestOnly + public Double getTestScore(Metric metric) { + return testScores.get(metric); + } + /** * Turns this class into a Cypher map, to be returned in a procedure YIELD field. * This is intentionally omitting the test scores. diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java index 8e06563dbeb..79ec73b7124 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.PropertyCursor; import org.neo4j.gds.api.schema.Direction; @@ -130,6 +131,9 @@ void splitSkewedGraph() { Optional.of(-1L), skewedGraphStore.nodes(), skewedGraphStore.nodes(), + skewedGraphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -147,6 +151,9 @@ void splitMultiGraph() { Optional.of(-1L), multiGraphStore.nodes(), multiGraphStore.nodes(), + multiGraphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -165,6 +172,9 @@ void split() { Optional.of(-1L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -174,7 +184,7 @@ void split() { var remainingRelationships = result.remainingRels().build(); // 2 positive selected reduces remaining assertEquals(3L, remainingRelationships.topology().elementCount()); - assertEquals(Direction.DIRECTED, remainingRelationships.direction()); + assertEquals(Direction.DIRECTED, remainingRelationships.relationshipSchemaEntry().direction()); assertFalse(remainingRelationships.topology().isMultiGraph()); assertThat(remainingRelationships.properties()).isNotEmpty(); assertRelInGraph(remainingRelationships, graph); @@ -199,7 +209,14 @@ void negativeEdgesShouldNotOverlapMasterGraph() { .build() .generate(); - var splitter = new DirectedEdgeSplitter(Optional.of(42L), huuuuugeDenseGraph, huuuuugeDenseGraph, 4); + var splitter = new DirectedEdgeSplitter(Optional.of(42L), + huuuuugeDenseGraph, + huuuuugeDenseGraph, + huuuuugeDenseGraph, + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), + 4 + ); var splitResult = splitter.splitPositiveExamples(huuuuugeDenseGraph, 0.9, Optional.empty()); var graph = GraphFactory.create( huuuuugeDenseGraph.idMap(), @@ -224,7 +241,15 @@ void negativeEdgesShouldNotOverlapMasterGraph() { @Test void negativeEdgeSampling() { - var splitter = new DirectedEdgeSplitter(Optional.of(42L), graphStore.nodes(), graphStore.nodes(), 4); + var splitter = new DirectedEdgeSplitter( + Optional.of(42L), + graphStore.nodes(), + graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), + 4 + ); var sum = 0; for (int i = 0; i < 100; i++) { @@ -241,8 +266,11 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { Collection targetNodeLabels = List.of(NodeLabel.of("C"), NodeLabel.of("D")); var splitter = new DirectedEdgeSplitter( Optional.of(1337L), + multiLabelGraphStore.nodes(), multiLabelGraphStore.getGraph(sourceNodeLabels), multiLabelGraphStore.getGraph(targetNodeLabels), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -269,7 +297,15 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { @Test void samplesWithinBounds() { - var splitter = new DirectedEdgeSplitter(Optional.of(42L), graphStore.nodes(), graphStore.nodes(), 4); + var splitter = new DirectedEdgeSplitter( + Optional.of(42L), + graphStore.nodes(), + graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), + 4 + ); assertEquals(1, splitter.samplesPerNode(1, 100, 10)); assertEquals(1, splitter.samplesPerNode(100, 1, 1)); @@ -277,7 +313,16 @@ void samplesWithinBounds() { @Test void shouldPreserveRelationshipWeights() { - var splitter = new DirectedEdgeSplitter(Optional.of(42L), graphStore.nodes(), graphStore.nodes(), 4); + var splitter = new DirectedEdgeSplitter( + Optional.of(42L), + graphStore.nodes(), + graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), + 4 + ); + EdgeSplitter.SplitResult split = splitter.splitPositiveExamples(graph, 0.01, Optional.of("foo")); var maybeProp = split.remainingRels().build().properties(); assertThat(maybeProp).isPresent(); diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/RandomNegativeSamplerTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/RandomNegativeSamplerTest.java index edc9afef118..bd86e9b287f 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/RandomNegativeSamplerTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/RandomNegativeSamplerTest.java @@ -20,6 +20,7 @@ package org.neo4j.gds.ml.splitting; import org.junit.jupiter.api.Test; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.Aggregation; @@ -72,10 +73,10 @@ void generateNegativeSamples() { RelationshipsBuilder testBuilder = new RelationshipsBuilderBuilder().nodes(graph).addPropertyConfig( GraphFactory.PropertyConfig.of("property", Aggregation.SINGLE, DefaultValue.forDouble()) - ).build(); + ).relationshipType(RelationshipType.of("TEST")).build(); RelationshipsBuilder trainBuilder = new RelationshipsBuilderBuilder().nodes(graph).addPropertyConfig( GraphFactory.PropertyConfig.of("property", Aggregation.SINGLE, DefaultValue.forDouble()) - ).build(); + ).relationshipType(RelationshipType.of("TRAIN")).build(); sampler.produceNegativeSamples(testBuilder, trainBuilder); diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java index c290c0397bb..2953286b5d6 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyCursor; @@ -96,6 +97,9 @@ void split() { Optional.of(1337L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -105,7 +109,7 @@ void split() { var remainingRelationships = result.remainingRels().build(); // 1 positive selected reduces remaining assertEquals(8L, remainingRelationships.topology().elementCount()); - assertEquals(Direction.UNDIRECTED, remainingRelationships.direction()); + assertEquals(Direction.UNDIRECTED, remainingRelationships.relationshipSchemaEntry().direction()); assertFalse(remainingRelationships.topology().isMultiGraph()); assertThat(remainingRelationships.properties()).isNotEmpty(); @@ -113,7 +117,7 @@ void split() { assertThat(selectedRelationships.topology()).satisfies(topology -> { assertRelSamplingProperties(selectedRelationships, graph); assertThat(topology.elementCount()).isEqualTo(1); - assertEquals(Direction.DIRECTED, selectedRelationships.direction()); + assertEquals(Direction.DIRECTED, selectedRelationships.relationshipSchemaEntry().direction()); assertFalse(topology.isMultiGraph()); }); } @@ -124,6 +128,9 @@ void splitMultiGraph() { Optional.of(-1L), multiGraphStore.nodes(), multiGraphStore.nodes(), + multiGraphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -152,6 +159,9 @@ void negativeEdgesShouldNotOverlapMasterGraph() { Optional.of(42L), huuuuugeDenseGraph, huuuuugeDenseGraph, + huuuuugeDenseGraph, + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); var splitResult = splitter.splitPositiveExamples(huuuuugeDenseGraph, 0.9, Optional.empty()); @@ -191,25 +201,31 @@ void shouldProduceDeterministicResult() { var splitResult1 = new UndirectedEdgeSplitter( Optional.of(12L), - graphStore.nodes(), - graphStore.nodes(), + graph.idMap(), + graph.idMap(), + graph.idMap(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var splitResult2 = new UndirectedEdgeSplitter( Optional.of(12L), - graphStore.nodes(), - graphStore.nodes(), + graph.idMap(), + graph.idMap(), + graph.idMap(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var remainingAreEqual = relationshipsAreEqual( - graph, + graph.idMap(), splitResult1.remainingRels().build(), splitResult2.remainingRels().build() ); assertTrue(remainingAreEqual); var holdoutAreEqual = relationshipsAreEqual( - graph, + graph.idMap(), splitResult1.selectedRels().build(), splitResult2.selectedRels().build() ); @@ -233,12 +249,18 @@ void shouldProduceNonDeterministicResult() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var splitResult2 = new UndirectedEdgeSplitter( Optional.of(117L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var remainingAreEqual = relationshipsAreEqual( @@ -262,6 +284,9 @@ void negativeEdgeSampling() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -282,6 +307,9 @@ void splitWithFilteringWithSameSourceTargetLabels() { Optional.of(1337L), graphStore.getGraph(NodeLabel.of("A")), graphStore.getGraph(NodeLabel.of("A")), + graphStore.getGraph(NodeLabel.of("A")), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -291,7 +319,7 @@ void splitWithFilteringWithSameSourceTargetLabels() { var remainingRelationships = result.remainingRels().build(); // 1 positive selected reduces remaining & 4 invalid relationships assertEquals(4L, remainingRelationships.topology().elementCount()); - assertEquals(Direction.UNDIRECTED, remainingRelationships.direction()); + assertEquals(Direction.UNDIRECTED, remainingRelationships.relationshipSchemaEntry().direction()); assertFalse(remainingRelationships.topology().isMultiGraph()); assertThat(remainingRelationships.properties()).isNotEmpty(); @@ -299,7 +327,7 @@ void splitWithFilteringWithSameSourceTargetLabels() { assertThat(selectedRelationships.topology()).satisfies(topology -> { assertRelSamplingProperties(selectedRelationships, graph); assertThat(topology.elementCount()).isEqualTo(1); - assertEquals(Direction.DIRECTED, selectedRelationships.direction()); + assertEquals(Direction.DIRECTED, selectedRelationships.relationshipSchemaEntry().direction()); assertFalse(topology.isMultiGraph()); }); @@ -313,8 +341,11 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { Collection targetNodeLabels = List.of(NodeLabel.of("C"), NodeLabel.of("D")); var splitter = new UndirectedEdgeSplitter( Optional.of(1337L), + multiLabelGraphStore.nodes(), multiLabelGraphStore.getGraph(sourceNodeLabels), multiLabelGraphStore.getGraph(targetNodeLabels), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -324,7 +355,7 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { var remainingRelationships = result.remainingRels().build(); // 2 positive selected reduces remaining & 4 invalid relationships assertEquals(2L, remainingRelationships.topology().elementCount()); - assertEquals(Direction.UNDIRECTED, remainingRelationships.direction()); + assertEquals(Direction.UNDIRECTED, remainingRelationships.relationshipSchemaEntry().direction()); assertFalse(remainingRelationships.topology().isMultiGraph()); assertThat(remainingRelationships.properties()).isNotEmpty(); assertRelInGraph(remainingRelationships, multiLabelGraph); @@ -333,7 +364,7 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { assertThat(selectedRelationships.topology()).satisfies(topology -> { assertRelSamplingProperties(selectedRelationships, multiLabelGraph); assertThat(topology.elementCount()).isEqualTo(2); - assertEquals(Direction.DIRECTED, selectedRelationships.direction()); + assertEquals(Direction.DIRECTED, selectedRelationships.relationshipSchemaEntry().direction()); assertFalse(topology.isMultiGraph()); }); @@ -346,6 +377,9 @@ void samplesWithinBounds() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -359,6 +393,9 @@ void shouldPreserveRelationshipWeights() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); EdgeSplitter.SplitResult split = splitter.splitPositiveExamples(graph, 0.01, Optional.of("foo")); @@ -386,6 +423,9 @@ void zeroNegativeSamples() { Optional.of(1337L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -402,7 +442,7 @@ private boolean relationshipsAreEqual(IdMap mapping, SingleTypeRelationships r1, return false; } - if (r1.direction() != r2.direction()) { + if (r1.relationshipSchemaEntry().direction() != r2.relationshipSchemaEntry().direction()) { return false; } diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java index bb824901872..de0e7e62550 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.Aggregation; @@ -30,6 +31,7 @@ import org.neo4j.gds.core.loading.construction.RelationshipsBuilderBuilder; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.ml.negativeSampling.UserInputNegativeSampler; @@ -39,11 +41,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import static org.neo4j.gds.ml.negativeSampling.NegativeSampler.NEGATIVE; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @GdlExtension class UserInputNegativeSamplerTest { - @GdlGraph(orientation = Orientation.UNDIRECTED) + @GdlGraph(orientation = Orientation.UNDIRECTED, idOffset = 5000) static String gdl = "(a1:A), " + "(a2:A), " + @@ -60,6 +63,9 @@ class UserInputNegativeSamplerTest { @Inject Graph graph; + @Inject + IdFunction idFunction; + @Test void generateNegativeSamples() { @@ -73,10 +79,10 @@ void generateNegativeSamples() { RelationshipsBuilder testBuilder = new RelationshipsBuilderBuilder().nodes(graph).addPropertyConfig( GraphFactory.PropertyConfig.of("property", Aggregation.SINGLE, DefaultValue.forDouble()) - ).build(); + ).relationshipType(RelationshipType.of("TEST")).build(); RelationshipsBuilder trainBuilder = new RelationshipsBuilderBuilder().nodes(graph).addPropertyConfig( GraphFactory.PropertyConfig.of("property", Aggregation.SINGLE, DefaultValue.forDouble()) - ).build(); + ).relationshipType(RelationshipType.of("TRAIN")).build(); sampler.produceNegativeSamples(testBuilder, trainBuilder); @@ -119,9 +125,11 @@ void shouldValidateNegativeExamplesRespectNodeLabels() { List.of(NodeLabel.of("A")) )) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("There is a relationship of negativeRelationshipType between nodes 0 and 1. " + + .hasMessageContaining(formatWithLocale("There is a relationship of negativeRelationshipType between nodes %d and %d. " + "The nodes have types [NodeLabel{name='A'}] and [NodeLabel{name='A'}]. " + - "However, they need to be between [NodeLabel{name='B'}] and [NodeLabel{name='A'}]." + "However, they need to be between [NodeLabel{name='B'}] and [NodeLabel{name='A'}].", + idFunction.of("a1"), idFunction.of("a2") + ) ); } diff --git a/ml/ml-core/src/main/java/org/neo4j/gds/ml/core/subgraph/SubGraph.java b/ml/ml-core/src/main/java/org/neo4j/gds/ml/core/subgraph/SubGraph.java index 3ad7285e082..362cc8c6710 100644 --- a/ml/ml-core/src/main/java/org/neo4j/gds/ml/core/subgraph/SubGraph.java +++ b/ml/ml-core/src/main/java/org/neo4j/gds/ml/core/subgraph/SubGraph.java @@ -69,24 +69,22 @@ public static List buildSubGraphs( } public static SubGraph buildSubGraph(long[] batchNodeIds, NeighborhoodFunction neighborhoodFunction, RelationshipWeights weightFunction) { - int[][] adjacency = new int[batchNodeIds.length][]; - int[] batchedNodeIds = new int[batchNodeIds.length]; + int[] mappedBatchNodeIds = new int[batchNodeIds.length]; // mapping original long-based nodeIds into consecutive int-based ids LocalIdMap idmap = new LocalIdMap(); // map the input node ids // this assures they are in consecutive order - for (long nodeId : batchNodeIds) { - idmap.toMapped(nodeId); - } - for (int nodeOffset = 0, nodeIdsLength = batchNodeIds.length; nodeOffset < nodeIdsLength; nodeOffset++) { - long nodeId = batchNodeIds[nodeOffset]; + int mappedNodeId = idmap.toMapped(batchNodeIds[nodeOffset]); + mappedBatchNodeIds[nodeOffset] = mappedNodeId; + } + int[][] adjacency = new int[idmap.size()][]; - batchedNodeIds[nodeOffset] = idmap.toMapped(nodeId); + for (int mappedNodeId = 0, mappedBatchIds = idmap.size(); mappedNodeId < mappedBatchIds; mappedNodeId++) { - var nodeNeighbors = neighborhoodFunction.sample(nodeId); + var nodeNeighbors = neighborhoodFunction.sample(idmap.toOriginal(mappedNodeId)); // map sampled neighbors into local id space // this also expands the id mapping as the neighbours could be not in the nodeIds[] @@ -94,10 +92,10 @@ public static SubGraph buildSubGraph(long[] batchNodeIds, NeighborhoodFunction n .mapToInt(idmap::toMapped) .toArray(); - adjacency[nodeOffset] = neighborInternalIds; + adjacency[mappedNodeId] = neighborInternalIds; } - return new SubGraph(adjacency, batchedNodeIds, idmap.originalIds(), weightFunction); + return new SubGraph(adjacency, mappedBatchNodeIds, idmap.originalIds(), weightFunction); } @Override diff --git a/ml/ml-core/src/test/java/org/neo4j/gds/ml/core/subgraph/SubGraphBuilderTest.java b/ml/ml-core/src/test/java/org/neo4j/gds/ml/core/subgraph/SubGraphBuilderTest.java index e290df76615..201d8143fc0 100644 --- a/ml/ml-core/src/test/java/org/neo4j/gds/ml/core/subgraph/SubGraphBuilderTest.java +++ b/ml/ml-core/src/test/java/org/neo4j/gds/ml/core/subgraph/SubGraphBuilderTest.java @@ -205,7 +205,7 @@ void shouldHandleDuplicatedNodes() { RelationshipWeights.UNWEIGHTED ); - assertEquals(6, subGraph.neighbors.length); + assertEquals(3, subGraph.neighbors.length); } @Test diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java index 8cb4740c2a7..3ccdeb79253 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java @@ -28,6 +28,10 @@ public interface ModelCatalog { + void registerListener(ModelCatalogListener listener); + + void unregisterListener(ModelCatalogListener listener); + void set(Model model); Model get( diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalogListener.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalogListener.java new file mode 100644 index 00000000000..8af4661122e --- /dev/null +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalogListener.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.model; + +public interface ModelCatalogListener { + + void onInsert(Model model); +} diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelHash.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelHash.java new file mode 100644 index 00000000000..eaf43967ef4 --- /dev/null +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelHash.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.model; + +import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.model.ModelConfig; + +import java.time.ZonedDateTime; + +@ValueClass +public +interface ModelHash { + String user(); + + String name(); + + ZonedDateTime timestamp(); + + static ModelHash of(ModelMetaData modelMetaData) { + return ImmutableModelHash.of( + modelMetaData.creator(), + modelMetaData.name(), + modelMetaData.creationTime() + ); + } + + static ModelHash of(Model model) { + return ImmutableModelHash.of( + model.creator(), + model.name(), + model.creationTime() + ); + } +} diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelMetaData.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelMetaData.java index a304d31bb7b..1336b8ccda0 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelMetaData.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelMetaData.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.model; +import org.immutables.value.Value; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.config.ToMapConvertible; @@ -27,6 +28,8 @@ import java.time.ZonedDateTime; import java.util.List; +import static org.neo4j.gds.core.model.Model.ALL_USERS; + @ValueClass public interface ModelMetaData { @@ -45,4 +48,9 @@ public interface ModelMetaData params, - TransactionalContextFactory contextFactory, - QueryExecutionEngine executionEngine, - QuerySubscriber subscriber - ) { - var convertedParams = ValueUtils.asMapValue(params); - var context = contextFactory.newContext(tx, query, convertedParams); - try { - return executionEngine.executeQuery(query, convertedParams, context, false, subscriber); - } catch (QueryExecutionKernelException e) { - throw e.asUserException(); - } - } - public static Result runQueryWithoutClosingTheResult( KernelTransaction tx, String query, diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 3035e9c6325..08b57c31b6d 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -31,7 +31,13 @@ public enum Neo4jVersion { V_5_1, V_5_2, V_5_3, - V_Dev; + V_5_4, + V_5_5, + V_5_6, + V_5_7, + V_5_8, + V_5_9, + V_RC; @Override public String toString() { @@ -44,18 +50,29 @@ public String toString() { return "5.2"; case V_5_3: return "5.3"; - case V_Dev: - return "dev"; + case V_5_4: + return "5.4"; + case V_5_5: + return "5.5"; + case V_5_6: + return "5.6"; + case V_5_7: + return "5.7"; + case V_5_8: + return "5.8"; + case V_5_9: + return "5.9"; + case V_RC: + return "rc"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } } public MajorMinorVersion semanticVersion() { - if (this == V_Dev) { - return ImmutableMajorMinorVersion.of(5, 4); + if (this == V_RC) { + return ImmutableMajorMinorVersion.of(5, 10); } - String version = toString(); var subVersions = version.split("\\."); @@ -122,8 +139,20 @@ static Neo4jVersion parse(String version) { return Neo4jVersion.V_5_2; } else if (minorVersion == 3) { return Neo4jVersion.V_5_3; - } else if (minorVersion > 3) { - return Neo4jVersion.V_Dev; + } else if (minorVersion == 4) { + return Neo4jVersion.V_5_4; + } else if (minorVersion == 5) { + return Neo4jVersion.V_5_5; + } else if (minorVersion == 6) { + return Neo4jVersion.V_5_6; + } else if (minorVersion == 7) { + return Neo4jVersion.V_5_7; + } else if (minorVersion == 8) { + return Neo4jVersion.V_5_8; + } else if (minorVersion == 9) { + return Neo4jVersion.V_5_9; + } else if (minorVersion == 10) { + return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index d07bb441c3c..247015c722d 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -40,11 +40,17 @@ class Neo4jVersionTest { "4.4.14, V_4_4", "5.1.0, V_5_1", "5.2.0, V_5_2", - "5.4.0, V_Dev", "5.1.0-dev, V_5_1", "5.2.0-dev, V_5_2", "5.2.0, V_5_2", "5.3.0, V_5_3", + "5.4.0, V_5_4", + "5.5.0, V_5_5", + "5.6.0, V_5_6", + "5.7.0, V_5_7", + "5.8.0, V_5_8", + "5.9.0, V_5_9", + "5.10.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); @@ -80,6 +86,12 @@ void shouldNotRespectVersionOverride() { "5.1.0, 5, 1", "5.2.0, 5, 2", "5.3.0, 5, 3", + "5.4.0, 5, 4", + "5.5.0, 5, 5", + "5.6.0, 5, 6", + "5.7.0, 5, 7", + "5.8.0, 5, 8", + "5.9.0, 5, 9", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); diff --git a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java index 5529baf2e75..6237e7a8cbf 100644 --- a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java +++ b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java @@ -26,9 +26,11 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; @@ -38,8 +40,21 @@ public final class OpenModelCatalog implements ModelCatalog { private final Map userCatalogs; + private final Set listeners; + public OpenModelCatalog() { this.userCatalogs = new ConcurrentHashMap<>(); + this.listeners = new HashSet<>(); + } + + @Override + public void registerListener(ModelCatalogListener listener) { + listeners.add(listener); + } + + @Override + public void unregisterListener(ModelCatalogListener listener) { + listeners.remove(listener); } @Override @@ -51,6 +66,8 @@ public void set(Model model) { userCatalog.set(model); return userCatalog; }); + + listeners.forEach(listener -> listener.onInsert(model)); } @Override diff --git a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java index e41f0a96b20..fc5b089699c 100644 --- a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java +++ b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.model; +import org.eclipse.collections.impl.Counter; import org.junit.jupiter.api.Test; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; @@ -118,6 +119,30 @@ void shouldStoreModelsPerType() { ); } + @Test + void shouldNotifyListeners() { + var counter = new Counter(); + modelCatalog.registerListener(model -> counter.increment()); + + var model = Model.of( + "testAlgo", + GRAPH_SCHEMA, + "testTrainData", + TestTrainConfig.of(USERNAME, "testModel"), + Map::of + ); + + assertThat(counter.getCount()).isEqualTo(0); + + modelCatalog.set(model); + + assertThat(counter.getCount()).isEqualTo(1); + + assertThatThrownBy(() -> modelCatalog.set(model)); + // not be called if the set is not successful + assertThat(counter.getCount()).isEqualTo(1); + } + @Test void shouldThrowWhenPublishing() { modelCatalog.set(TEST_MODEL); diff --git a/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java b/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java index 15b55c5b63c..741656ba776 100644 --- a/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java +++ b/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java @@ -52,6 +52,7 @@ class OpenGdsProcedureSmokeTest extends BaseProcTest { "gds.alpha.graph.sample.rwr", "gds.alpha.create.cypherdb", + "gds.alpha.drop.cypherdb", "gds.alpha.allShortestPaths.stream", @@ -532,7 +533,7 @@ void countShouldMatch() { ); // If you find yourself updating this count, please also update the count in SmokeTest.kt - int expectedCount = 377; + int expectedCount = 378; assertEquals( expectedCount, registeredProcedures.size(), diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkFeaturesAndLabelsExtractor.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkFeaturesAndLabelsExtractor.java index 289f1a4a1ae..63f087cf9a5 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkFeaturesAndLabelsExtractor.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkFeaturesAndLabelsExtractor.java @@ -64,7 +64,7 @@ static MemoryEstimation estimate( .times(relSetSizeExtractor.applyAsLong(graphDim.relationshipCounts())) .add(MemoryUsage.sizeOfInstance(HugeObjectArray.class))) .perGraphDimension( - setDesc + "relationship targets", + setDesc + " relationship targets", (graphDim, threads) -> MemoryRange.of( HugeIntArray.memoryEstimation(relSetSizeExtractor.applyAsLong(graphDim.relationshipCounts())) ) diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java index 803113deff0..fcb42a5c8fd 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.ElementProjection; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; @@ -41,6 +42,7 @@ import org.neo4j.gds.ml.splitting.EdgeSplitter; import org.neo4j.gds.ml.splitting.UndirectedEdgeSplitter; +import java.util.Collection; import java.util.List; import java.util.Optional; @@ -102,29 +104,47 @@ public void splitAndSampleRelationships( var targetLabels = ElementTypeValidator.resolve(graphStore, List.of(trainConfig.targetNodeLabel())); IdMap sourceNodes = graphStore.getGraph(sourceLabels); IdMap targetNodes = graphStore.getGraph(targetLabels); + Collection sourceAndTargetNodeLabels = trainConfig.nodeLabelIdentifiers(graphStore); var graph = graphStore.getGraph( - trainConfig.nodeLabelIdentifiers(graphStore), + sourceAndTargetNodeLabels, trainConfig.internalRelationshipTypes(graphStore), relationshipWeightProperty); // Relationship sets: test, train, feature-input, test-complement. The nodes are always the same. // 1. Split base graph into test, test-complement terminationFlag.assertRunning(); - var testSplitResult = split(sourceNodes, targetNodes, graph, relationshipWeightProperty, splitConfig.testComplementRelationshipType(), splitConfig.testFraction()); + var testSplitResult = split( + sourceNodes, + targetNodes, + graph, + relationshipWeightProperty, + splitConfig.testRelationshipType(), + splitConfig.testComplementRelationshipType(), + splitConfig.testFraction() + ); // 2. Split test-complement into (labeled) train and feature-input. var testComplementGraph = graphStore.getGraph( - trainConfig.nodeLabelIdentifiers(graphStore), + sourceAndTargetNodeLabels, List.of(splitConfig.testComplementRelationshipType()), relationshipWeightProperty ); terminationFlag.assertRunning(); - var trainSplitResult = split(sourceNodes, targetNodes, testComplementGraph, relationshipWeightProperty, splitConfig.featureInputRelationshipType(), splitConfig.trainFraction()); + var trainSplitResult = split( + sourceNodes, + targetNodes, + testComplementGraph, + relationshipWeightProperty, + splitConfig.trainRelationshipType(), + splitConfig.featureInputRelationshipType(), + splitConfig.trainFraction() + ); // 3. add negative examples to test and train NegativeSampler negativeSampler = NegativeSampler.of( graphStore, graph, + sourceAndTargetNodeLabels, splitConfig.negativeRelationshipType(), splitConfig.negativeSamplingRatio(), testSplitResult.selectedRelCount(), @@ -140,8 +160,8 @@ public void splitAndSampleRelationships( negativeSampler.produceNegativeSamples(testSplitResult.selectedRels(), trainSplitResult.selectedRels()); // 4. Update graphStore with (positive+negative) 'TEST' and 'TRAIN' edges - graphStore.addRelationshipType(splitConfig.testRelationshipType(), testSplitResult.selectedRels().build()); - graphStore.addRelationshipType(splitConfig.trainRelationshipType(), trainSplitResult.selectedRels().build()); + graphStore.addRelationshipType(testSplitResult.selectedRels().build()); + graphStore.addRelationshipType(trainSplitResult.selectedRels().build()); validateTestSplit(graphStore); validateTrainSplit(graphStore); @@ -150,14 +170,25 @@ public void splitAndSampleRelationships( progressTracker.endSubTask("Split relationships"); } - private EdgeSplitter.SplitResult split(IdMap sourceNodes, IdMap targetNodes, Graph graph, Optional relationshipWeightProperty, RelationshipType remainingRelType, double selectedFraction) { + private EdgeSplitter.SplitResult split( + IdMap sourceNodes, + IdMap targetNodes, + Graph graph, + Optional relationshipWeightProperty, + RelationshipType selectedRelType, + RelationshipType remainingRelType, + double selectedFraction + ) { if (!graph.schema().isUndirected()) { throw new IllegalArgumentException("EdgeSplitter requires graph to be UNDIRECTED"); } var splitter = new UndirectedEdgeSplitter( trainConfig.randomSeed(), + graphStore.nodes(), sourceNodes, targetNodes, + selectedRelType, + remainingRelType, ConcurrencyConfig.DEFAULT_CONCURRENCY ); @@ -168,7 +199,7 @@ private EdgeSplitter.SplitResult split(IdMap sourceNodes, IdMap targetNodes, Gra ); var remainingRels = splitResult.remainingRels().build(); - graphStore.addRelationshipType(remainingRelType, remainingRels); + graphStore.addRelationshipType(remainingRels); return splitResult; } diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrain.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrain.java index 702723025d4..6f6375ddfe0 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrain.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrain.java @@ -207,6 +207,7 @@ private void findBestModelCandidate( crossValidation.selectModel( trainRelationshipIds, trainData.labels()::get, + //LP always have 2 classes 0,1 the original ids happen to be the same as internal new TreeSet<>(classIdMap.originalIdsList()), trainingStatistics, modelCandidates diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrain.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrain.java index 4a3972a26a0..d3ebf2dc16d 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrain.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrain.java @@ -70,6 +70,7 @@ import java.util.TreeSet; import java.util.function.LongUnaryOperator; import java.util.stream.Collectors; +import java.util.stream.LongStream; import static org.neo4j.gds.core.utils.mem.MemoryEstimations.delegateEstimation; import static org.neo4j.gds.core.utils.mem.MemoryEstimations.maxEstimation; @@ -374,10 +375,10 @@ private void findBestModelCandidate(ReadOnlyHugeLongArray trainNodeIds, Features trainConfig.randomSeed() ); - var sortedClassIds = new TreeSet(); - for (long clazz : classCounts.keys()) { - sortedClassIds.add(clazz); - } + var sortedClassIds = LongStream + .range(0, classCounts.size()) + .boxed() + .collect(Collectors.toCollection(TreeSet::new)); crossValidation.selectModel( trainNodeIds, diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java index 6bca527a1da..51ed227b268 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java @@ -36,9 +36,11 @@ import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.ml.pipeline.linkPipeline.LinkPredictionSplitConfigImpl; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -54,7 +56,7 @@ @GdlExtension class LinkPredictionRelationshipSamplerTest { - @GdlGraph(orientation = Orientation.UNDIRECTED) + @GdlGraph(orientation = Orientation.UNDIRECTED, idOffset = 1337) private static final String GRAPH = "CREATE " + "(a:N {scalar: 0, array: [-1.0, -2.0, 1.0, 1.0, 3.0]}), " + @@ -64,6 +66,8 @@ class LinkPredictionRelationshipSamplerTest { "(e:N {scalar: 1, array: [-2.0, 1.0, 2.0, 1.0, -1.0]}), " + "(f:N {scalar: 0, array: [-1.0, -3.0, 1.0, 2.0, 2.0]}), " + "(g:N {scalar: 1, array: [3.0, 1.0, -3.0, 3.0, 1.0]}), " + + // leaving some id gap between nodes + "(:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), ".repeat(20) + "(h:N {scalar: 3, array: [-1.0, 3.0, 2.0, 1.0, -3.0]}), " + "(i:N {scalar: 3, array: [4.0, 1.0, 1.0, 2.0, 1.0]}), " + "(j:N {scalar: 4, array: [1.0, -4.0, 2.0, -2.0, 2.0]}), " + @@ -72,7 +76,7 @@ class LinkPredictionRelationshipSamplerTest { "(m:N {scalar: 0, array: [4.0, 4.0, 1.0, 1.0, 1.0]}), " + "(n:N {scalar: 3, array: [1.0, -2.0, 3.0, 2.0, 3.0]}), " + "(o:N {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "" + + "(a)-[:REL {weight: 2.0}]->(b), " + "(a)-[:REL {weight: 1.0}]->(c), " + "(b)-[:REL {weight: 3.0}]->(c), " + @@ -96,6 +100,9 @@ class LinkPredictionRelationshipSamplerTest { @Inject GraphStore graphStore; + @Inject + IdFunction idFunction; + @GdlGraph(graphNamePrefix = "multi", orientation = Orientation.UNDIRECTED) private static final String MULTI_GRAPH = "CREATE " + @@ -363,7 +370,7 @@ void splitWithSpecifiedNegativeRelationships() { .negativeRelationshipType("NEGATIVE") // 3 total .build(); - var trainConfig = createTrainConfig("REL", "*", "N", -1337L); + var trainConfig = createTrainConfig("REL", "N", "N", -1337L); var relationshipSplitter = new LinkPredictionRelationshipSampler( graphStore, @@ -386,6 +393,23 @@ void splitWithSpecifiedNegativeRelationships() { //8 * 0.5 = 4 positive, 1 negative assertThat(trainGraphSize).isEqualTo(5); assertThat(featureInputGraphSize).isEqualTo(8); - + var outGraph = graphStore.getGraph(trainConfig.nodeLabelIdentifiers(graphStore), List.of(splitConfig.testRelationshipType(), splitConfig.trainRelationshipType()), Optional.of("label")); + + var negativeRelSpace = graphStore.getGraph(RelationshipType.of("NEGATIVE")); + var positiveRelSpace = graphStore.getGraph(RelationshipType.of("REL")); + + outGraph.forEachNode(nodeId -> { + outGraph.forEachRelationship(nodeId, Double.NaN, (s,t, w) -> { + if (w == 1.0) { + assertThat(positiveRelSpace.exists(outGraph.toRootNodeId(s), outGraph.toRootNodeId(t))).isTrue(); + } + if (w == 0.0) { + assertThat(negativeRelSpace.exists(outGraph.toRootNodeId(s), outGraph.toRootNodeId(t))).isTrue(); + } + return true; + }); + return true; + } + ); } } diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationToModelConverterTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationToModelConverterTest.java index 428f50feae4..8beb363b5bd 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationToModelConverterTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationToModelConverterTest.java @@ -25,14 +25,14 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.collections.LongMultiSet; import org.neo4j.gds.ml.core.subgraph.LocalIdMap; -import org.neo4j.gds.ml.metrics.ModelCandidateStats; import org.neo4j.gds.ml.metrics.EvaluationScores; +import org.neo4j.gds.ml.metrics.ModelCandidateStats; import org.neo4j.gds.ml.metrics.classification.ClassificationMetricSpecification; import org.neo4j.gds.ml.models.logisticregression.LogisticRegressionClassifier; import org.neo4j.gds.ml.models.logisticregression.LogisticRegressionData; @@ -94,9 +94,10 @@ void convertsModel() { .build(); var converter = new NodeClassificationToModelConverter(pipeline, config); - var originalSchema = ImmutableGraphSchema.builder() - .nodeSchema(NodeSchema.empty().addLabel(NodeLabel.of("M"))) - .relationshipSchema(RelationshipSchema.empty().addRelationshipType(RelationshipType.of("R"), Direction.UNDIRECTED)) + var originalSchema = MutableGraphSchema.builder() + .nodeSchema(MutableNodeSchema.empty().addLabel(NodeLabel.of("M"))) + .relationshipSchema(MutableRelationshipSchema + .empty().addRelationshipType(RelationshipType.of("R"), Direction.UNDIRECTED)) .putGraphProperty("array", PropertySchema.of("array", ValueType.DOUBLE_ARRAY)) .putGraphProperty("scalar", PropertySchema.of("scalar", ValueType.DOUBLE)) .build(); diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrainClassValueInvarianceTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrainClassValueInvarianceTest.java new file mode 100644 index 00000000000..3da0f72c518 --- /dev/null +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrainClassValueInvarianceTest.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.ml.pipeline.nodePipeline.classification.train; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.executor.ExecutionContext; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.ml.metrics.classification.Accuracy; +import org.neo4j.gds.ml.metrics.classification.ClassificationMetricSpecification; +import org.neo4j.gds.ml.metrics.classification.GlobalAccuracy; +import org.neo4j.gds.ml.models.logisticregression.LogisticRegressionTrainConfigImpl; +import org.neo4j.gds.ml.pipeline.nodePipeline.NodeFeatureProducer; +import org.neo4j.gds.ml.pipeline.nodePipeline.NodeFeatureStep; +import org.neo4j.gds.ml.pipeline.nodePipeline.classification.NodeClassificationTrainingPipeline; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@GdlExtension +public class NodeClassificationTrainClassValueInvarianceTest { + + private static final String GRAPH_NAME_1 = "G11"; + + @GdlGraph(graphNamePrefix = "nodes1") + private static final String DB_QUERY1 = + "CREATE " + + " (a1:N {bananas: 100.0, arrayProperty: [1.2, 1.2], a: 1.2, b: 1.2, t: 0})" + + ", (a2:N {bananas: 100.0, arrayProperty: [2.8, 2.5], a: 2.8, b: 2.5, t: 0})" + + ", (a3:N {bananas: 100.0, arrayProperty: [3.3, 0.5], a: 3.3, b: 0.5, t: 0})" + + ", (a4:N {bananas: 100.0, arrayProperty: [1.0, 0.5], a: 1.0, b: 0.5, t: 0})" + + ", (a5:N {bananas: 100.0, arrayProperty: [1.32, 0.5], a: 1.32, b: 0.5, t: 0})" + + ", (a6:N {bananas: 100.0, arrayProperty: [1.3, 1.5], a: 1.3, b: 1.5, t: 1})" + + ", (a7:N {bananas: 100.0, arrayProperty: [5.3, 10.5], a: 5.3, b: 10.5, t: 1})" + + ", (a8:N {bananas: 100.0, arrayProperty: [1.3, 2.5], a: 1.3, b: 2.5, t: 1})" + + ", (a9:N {bananas: 100.0, arrayProperty: [0.0, 66.8], a: 0.0, b: 66.8, t: 1})" + + ", (a10:N {bananas: 100.0, arrayProperty: [0.1, 2.8], a: 0.1, b: 2.8, t: 1})" + + ", (a11:N {bananas: 100.0, arrayProperty: [0.66, 2.8], a: 0.66, b: 2.8, t: 1})" + + ", (a12:N {bananas: 100.0, arrayProperty: [2.0, 10.8], a: 2.0, b: 10.8, t: 1})" + + ", (a13:N {bananas: 100.0, arrayProperty: [5.0, 7.8], a: 5.0, b: 7.8, t: 2})" + + ", (a14:N {bananas: 100.0, arrayProperty: [4.0, 5.8], a: 4.0, b: 5.8, t: 2})" + + ", (a15:N {bananas: 100.0, arrayProperty: [1.0, 0.9], a: 1.0, b: 0.9, t: 2})"; + + @Inject + private GraphStore nodes1GraphStore; + + private static final String GRAPH_NAME_2 = "G2"; + + @GdlGraph(graphNamePrefix = "nodes2") + private static final String DB_QUERY2 = + "CREATE " + + " (a1:N {bananas: 100.0, arrayProperty: [1.2, 1.2], a: 1.2, b: 1.2, t: 0})" + + ", (a2:N {bananas: 100.0, arrayProperty: [2.8, 2.5], a: 2.8, b: 2.5, t: 0})" + + ", (a3:N {bananas: 100.0, arrayProperty: [3.3, 0.5], a: 3.3, b: 0.5, t: 0})" + + ", (a4:N {bananas: 100.0, arrayProperty: [1.0, 0.5], a: 1.0, b: 0.5, t: 0})" + + ", (a5:N {bananas: 100.0, arrayProperty: [1.32, 0.5], a: 1.32, b: 0.5, t: 0})" + + ", (a6:N {bananas: 100.0, arrayProperty: [1.3, 1.5], a: 1.3, b: 1.5, t: 222})" + + ", (a7:N {bananas: 100.0, arrayProperty: [5.3, 10.5], a: 5.3, b: 10.5, t: 222})" + + ", (a8:N {bananas: 100.0, arrayProperty: [1.3, 2.5], a: 1.3, b: 2.5, t: 222})" + + ", (a9:N {bananas: 100.0, arrayProperty: [0.0, 66.8], a: 0.0, b: 66.8, t: 222})" + + ", (a10:N {bananas: 100.0, arrayProperty: [0.1, 2.8], a: 0.1, b: 2.8, t: 222})" + + ", (a11:N {bananas: 100.0, arrayProperty: [0.66, 2.8], a: 0.66, b: 2.8, t: 222})" + + ", (a12:N {bananas: 100.0, arrayProperty: [2.0, 10.8], a: 2.0, b: 10.8, t: 222})" + + ", (a13:N {bananas: 100.0, arrayProperty: [5.0, 7.8], a: 5.0, b: 7.8, t: 333})" + + ", (a14:N {bananas: 100.0, arrayProperty: [4.0, 5.8], a: 4.0, b: 5.8, t: 333})" + + ", (a15:N {bananas: 100.0, arrayProperty: [1.0, 0.9], a: 1.0, b: 0.9, t: 333})"; + + @Inject + private GraphStore nodes2GraphStore; + + /** + * This tests that the specific class values do not matter, as long as the ordering is the same. + * However, if the *ordering* of the class values differ, the splits in cross-validation could differ, resulting in different accuracies. + */ + @Test + void trainWithDifferentClassValues() { + var pipeline = new NodeClassificationTrainingPipeline(); + pipeline.addFeatureStep(NodeFeatureStep.of("a")); + pipeline.addFeatureStep(NodeFeatureStep.of("b")); + + var lrTrainerConfig = LogisticRegressionTrainConfigImpl.builder().build(); + pipeline.addTrainerConfig(lrTrainerConfig); + + var accuracyMetricSpec = ClassificationMetricSpecification.Parser.parse("accuracy"); + var accuracyPerClassMetricSpec = ClassificationMetricSpecification.Parser.parse("accuracy(class=*)"); + + var config01 = createConfig("model1", GRAPH_NAME_1, List.of(accuracyMetricSpec, accuracyPerClassMetricSpec), 1L); + var ncTrain01 = createWithExecutionContext( + nodes1GraphStore, + pipeline, + config01, + ProgressTracker.NULL_TRACKER + ); + var result01 = ncTrain01.run(); + assertThat(result01.classifier().data().featureDimension()).isEqualTo(2); + + //Run with graph that have class values 0 and 2 + var config02 = createConfig("model2", GRAPH_NAME_2, List.of(accuracyMetricSpec, accuracyPerClassMetricSpec), 1L); + var ncTrain02 = createWithExecutionContext( + nodes2GraphStore, + pipeline, + config02, + ProgressTracker.NULL_TRACKER + ); + var result02 = ncTrain02.run(); + assertThat(result01.classifier().data().featureDimension()).isEqualTo(2); + + var globalAccuracy = new GlobalAccuracy(); + var accuracyForClass1 = new Accuracy(0, 0); + var accuracyForClass2 = new Accuracy(1, 1); + var accuracyForClass3 = new Accuracy(2, 2); + + var accuracyForClass0 = new Accuracy(0, 0); + var accuracyForClass222 = new Accuracy(222, 1); + var accuracyForClass333 = new Accuracy(333, 2); + + assertThat(result01.trainingStatistics().getTrainStats(globalAccuracy)).isEqualTo(result02.trainingStatistics().getTrainStats(globalAccuracy)); + assertThat(result01.trainingStatistics().getTrainStats(accuracyForClass1)).isEqualTo(result02.trainingStatistics().getTrainStats(accuracyForClass0)); + assertThat(result01.trainingStatistics().getTrainStats(accuracyForClass2)).isEqualTo(result02.trainingStatistics().getTrainStats(accuracyForClass222)); + assertThat(result01.trainingStatistics().getTrainStats(accuracyForClass3)).isEqualTo(result02.trainingStatistics().getTrainStats(accuracyForClass333)); + + assertThat(result01.trainingStatistics().getValidationStats(globalAccuracy)).isEqualTo(result02.trainingStatistics().getValidationStats(globalAccuracy)); + assertThat(result01.trainingStatistics().getValidationStats(accuracyForClass1)).isEqualTo(result02.trainingStatistics().getValidationStats(accuracyForClass0)); + assertThat(result01.trainingStatistics().getValidationStats(accuracyForClass2)).isEqualTo(result02.trainingStatistics().getValidationStats(accuracyForClass222)); + assertThat(result01.trainingStatistics().getValidationStats(accuracyForClass3)).isEqualTo(result02.trainingStatistics().getValidationStats(accuracyForClass333)); + + assertThat(result01.trainingStatistics().getTestScore(globalAccuracy)).isEqualTo(result02.trainingStatistics().getTestScore(globalAccuracy)); + assertThat(result01.trainingStatistics().getTestScore(accuracyForClass1)).isEqualTo(result02.trainingStatistics().getTestScore(accuracyForClass0)); + assertThat(result01.trainingStatistics().getTestScore(accuracyForClass2)).isEqualTo(result02.trainingStatistics().getTestScore(accuracyForClass222)); + assertThat(result01.trainingStatistics().getTestScore(accuracyForClass3)).isEqualTo(result02.trainingStatistics().getTestScore(accuracyForClass333)); + + } + + private NodeClassificationPipelineTrainConfig createConfig( + String modelName, + String graphName, + List metricSpecification, + long randomSeed + ) { + return NodeClassificationPipelineTrainConfigImpl.builder() + .pipeline("") + .graphName(graphName) + .modelUser("DUMMY") + .modelName(modelName) + .concurrency(1) + .randomSeed(randomSeed) + .targetProperty("t") + .metrics(metricSpecification) + .build(); + } + + static NodeClassificationTrain createWithExecutionContext( + GraphStore graphStore, + NodeClassificationTrainingPipeline pipeline, + NodeClassificationPipelineTrainConfig config, + ProgressTracker progressTracker + ) { + var nodeFeatureProducer = NodeFeatureProducer.create(graphStore, config, ExecutionContext.EMPTY, progressTracker); + return NodeClassificationTrain.create( + graphStore, + pipeline, + config, + nodeFeatureProducer, + progressTracker + ); + } + +} diff --git a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java index d129113bc7b..da056385136 100644 --- a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java +++ b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java @@ -50,7 +50,10 @@ final class PregelValidation { private final Elements elementUtils; // Represents the PregelComputation interface - private final TypeMirror pregelComputation; + private final TypeMirror basePregelComputation; + + private final TypeMirror bidirectionalPregelComputation; + // Represents the PregelProcedureConfig interface private final TypeMirror pregelProcedureConfig; @@ -58,9 +61,12 @@ final class PregelValidation { this.messager = messager; this.typeUtils = typeUtils; this.elementUtils = elementUtils; - this.pregelComputation = MoreTypes.asDeclared( + this.basePregelComputation = MoreTypes.asDeclared( typeUtils.erasure(elementUtils.getTypeElement(BasePregelComputation.class.getName()).asType()) ); + this.bidirectionalPregelComputation = MoreTypes.asDeclared( + typeUtils.erasure(elementUtils.getTypeElement(BidirectionalPregelComputation.class.getName()).asType()) + ); this.pregelProcedureConfig = MoreTypes.asDeclared(elementUtils .getTypeElement(PregelProcedureConfig.class.getName()) .asType()); @@ -69,7 +75,7 @@ final class PregelValidation { Optional validate(Element pregelElement) { if ( !isClass(pregelElement) || - !isPregelComputation(pregelElement) || + !isBasePregelComputation(pregelElement) || !isPregelProcedureConfig(pregelElement) || !hasEmptyConstructor(pregelElement) || !configHasFactoryMethod(pregelElement) @@ -92,7 +98,8 @@ Optional validate(Element pregelElement) { configTypeName, procedure.name(), procedure.modes(), - maybeDescription + maybeDescription, + requiresInverseIndex(pregelElement) )); } @@ -108,17 +115,17 @@ private boolean isClass(Element pregelElement) { return isClass; } - private Optional pregelComputation(Element pregelElement) { + private Optional pregelComputation(Element pregelElement, TypeMirror computationInterface) { // TODO: this check needs to bubble up the inheritance tree return MoreElements.asType(pregelElement).getInterfaces().stream() .map(MoreTypes::asDeclared) - .filter(declaredType -> typeUtils.isSubtype(declaredType, pregelComputation)) + .filter(declaredType -> typeUtils.isSubtype(declaredType, computationInterface)) .findFirst(); } - private boolean isPregelComputation(Element pregelElement) { + private boolean isBasePregelComputation(Element pregelElement) { var pregelTypeElement = MoreElements.asType(pregelElement); - var maybeInterface = pregelComputation(pregelElement); + var maybeInterface = pregelComputation(pregelElement, basePregelComputation); boolean isPregelComputation = maybeInterface.isPresent(); if (!isPregelComputation) { @@ -131,6 +138,11 @@ private boolean isPregelComputation(Element pregelElement) { return isPregelComputation; } + private boolean requiresInverseIndex(Element pregelElement) { + var maybeInterface = pregelComputation(pregelElement, bidirectionalPregelComputation); + return maybeInterface.isPresent(); + } + private boolean isPregelProcedureConfig(Element pregelElement) { var config = config(pregelElement); @@ -198,7 +210,7 @@ private boolean configHasFactoryMethod(Element pregelElement) { } private TypeMirror config(Element pregelElement) { - return pregelComputation(pregelElement) + return pregelComputation(pregelElement, basePregelComputation) .map(declaredType -> declaredType.getTypeArguments().get(0)) .orElseThrow(() -> new IllegalStateException("Could not find a pregel computation")); } @@ -218,6 +230,8 @@ interface Spec { GDSMode[] procedureModes(); Optional description(); + + boolean requiresInverseIndex(); } } diff --git a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/ProcedureGenerator.java b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/ProcedureGenerator.java index e0dad2c1e34..537c0b52fd9 100644 --- a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/ProcedureGenerator.java +++ b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/ProcedureGenerator.java @@ -36,6 +36,8 @@ import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.executor.ExecutionMode; import org.neo4j.gds.executor.GdsCallable; +import org.neo4j.gds.executor.validation.ValidationConfiguration; +import org.neo4j.gds.pregel.proc.PregelBaseProc; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Mode; @@ -98,6 +100,11 @@ TypeSpec typeSpec() { typeSpecBuilder.addMethod(procResultMethod()); typeSpecBuilder.addMethod(newConfigMethod()); typeSpecBuilder.addMethod(algorithmFactoryMethod(algorithmClassName)); + + if (pregelSpec.requiresInverseIndex()) { + typeSpecBuilder.addMethod(validationConfigMethod()); + } + return typeSpecBuilder.build(); } @@ -281,4 +288,16 @@ private MethodSpec algorithmFactoryMethod(ClassName algorithmClassName) { .addStatement("return $L", anonymousFactoryType) .build(); } + + private MethodSpec validationConfigMethod() { + return MethodSpec.methodBuilder("validationConfig") + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .returns(ParameterizedTypeName.get( + ClassName.get(ValidationConfiguration.class), + pregelSpec.configTypeName() + )) + .addStatement("return $T.ensureIndexValidation(log, taskRegistryFactory)", PregelBaseProc.class) + .build(); + } } diff --git a/pregel-proc-generator/src/test/java/org/neo4j/gds/beta/pregel/PregelProcessorTest.java b/pregel-proc-generator/src/test/java/org/neo4j/gds/beta/pregel/PregelProcessorTest.java index 532c29fefe9..51b87795800 100644 --- a/pregel-proc-generator/src/test/java/org/neo4j/gds/beta/pregel/PregelProcessorTest.java +++ b/pregel-proc-generator/src/test/java/org/neo4j/gds/beta/pregel/PregelProcessorTest.java @@ -62,6 +62,22 @@ void positiveTest(String className) { ); } + @ParameterizedTest + @ValueSource(strings = { + "BidirectionalComputation" + }) + void positiveBiTest(String className) { + assertAbout(javaSource()) + .that(forResource(String.format("positive/%s.java", className))) + .processedWith(new PregelProcessor()) + .compilesWithoutError() + .and() + .generatesSources( + loadExpectedFile(formatWithLocale("expected/%sStreamProc.java", className)), + loadExpectedFile(formatWithLocale("expected/%sAlgorithm.java", className)) + ); + } + @Test void baseClassMustBeAClass() { runNegativeTest( diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java new file mode 100644 index 00000000000..87f984229c4 --- /dev/null +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.beta.pregel.cc; + +import javax.annotation.processing.Generated; +import org.neo4j.gds.Algorithm; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.beta.pregel.Pregel; +import org.neo4j.gds.beta.pregel.PregelProcedureConfig; +import org.neo4j.gds.beta.pregel.PregelResult; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.utils.TerminationFlag; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; + +@Generated("org.neo4j.gds.beta.pregel.PregelProcessor") +public final class BidirectionalComputationAlgorithm extends Algorithm { + private final Pregel pregelJob; + + BidirectionalComputationAlgorithm(Graph graph, PregelProcedureConfig configuration, + ProgressTracker progressTracker) { + super(progressTracker); + this.pregelJob = Pregel.create(graph, configuration, new BidirectionalComputation(), Pools.DEFAULT, progressTracker); + } + + @Override + public void setTerminationFlag(TerminationFlag terminationFlag) { + super.setTerminationFlag(terminationFlag); + pregelJob.setTerminationFlag(terminationFlag); + } + + @Override + public PregelResult compute() { + return pregelJob.run(); + } + + @Override + public void release() { + pregelJob.release(); + } +} diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java new file mode 100644 index 00000000000..f1d3d2eea12 --- /dev/null +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.beta.pregel.cc; + +import java.util.Map; +import java.util.stream.Stream; +import javax.annotation.processing.Generated; +import org.neo4j.gds.BaseProc; +import org.neo4j.gds.GraphAlgorithmFactory; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.beta.pregel.Pregel; +import org.neo4j.gds.beta.pregel.PregelProcedureConfig; +import org.neo4j.gds.core.CypherMapWrapper; +import org.neo4j.gds.core.utils.mem.MemoryEstimation; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.Task; +import org.neo4j.gds.executor.ExecutionMode; +import org.neo4j.gds.executor.GdsCallable; +import org.neo4j.gds.executor.validation.ValidationConfiguration; +import org.neo4j.gds.pregel.proc.PregelBaseProc; +import org.neo4j.gds.pregel.proc.PregelStreamProc; +import org.neo4j.gds.pregel.proc.PregelStreamResult; +import org.neo4j.gds.results.MemoryEstimateResult; +import org.neo4j.procedure.Description; +import org.neo4j.procedure.Mode; +import org.neo4j.procedure.Name; +import org.neo4j.procedure.Procedure; + +@GdsCallable( + name = "gds.pregel.bidirectionalTest.stream", + executionMode = ExecutionMode.STREAM, + description = "Bidirectional Test computation description" +) +@Generated("org.neo4j.gds.beta.pregel.PregelProcessor") +public final class BidirectionalComputationStreamProc extends PregelStreamProc { + @Procedure( + name = "gds.pregel.bidirectionalTest.stream", + mode = Mode.READ + ) + @Description("Bidirectional Test computation description") + public Stream stream(@Name("graphName") String graphName, + @Name(value = "configuration", defaultValue = "{}") Map configuration) { + return stream(compute(graphName, configuration)); + } + + @Procedure( + name = "gds.pregel.bidirectionalTest.stream.estimate", + mode = Mode.READ + ) + @Description(BaseProc.ESTIMATE_DESCRIPTION) + public Stream estimate( + @Name("graphNameOrConfiguration") Object graphNameOrConfiguration, + @Name("algoConfiguration") Map algoConfiguration) { + return computeEstimate(graphNameOrConfiguration, algoConfiguration); + } + + @Override + protected PregelStreamResult streamResult(long originalNodeId, long internalNodeId, + NodePropertyValues nodePropertyValues) { + throw new UnsupportedOperationException(); + } + + @Override + protected PregelProcedureConfig newConfig(String username, CypherMapWrapper config) { + return PregelProcedureConfig.of(config); + } + + @Override + public GraphAlgorithmFactory algorithmFactory( + ) { + return new GraphAlgorithmFactory() { + @Override + public BidirectionalComputationAlgorithm build(Graph graph, + PregelProcedureConfig configuration, ProgressTracker progressTracker) { + return new BidirectionalComputationAlgorithm(graph, configuration, progressTracker); + } + + @Override + public String taskName() { + return BidirectionalComputationAlgorithm.class.getSimpleName(); + } + + @Override + public Task progressTask(Graph graph, PregelProcedureConfig configuration) { + return Pregel.progressTask(graph, configuration); + } + + @Override + public MemoryEstimation memoryEstimation(PregelProcedureConfig configuration) { + var computation = new BidirectionalComputation(); + return Pregel.memoryEstimation(computation.schema(configuration), computation.reducer().isEmpty(), configuration.isAsynchronous()); + } + }; + } + + @Override + public ValidationConfiguration validationConfig() { + return PregelBaseProc.ensureIndexValidation(log, taskRegistryFactory); + } +} diff --git a/pregel-proc-generator/src/test/resources/positive/BidirectionalComputation.java b/pregel-proc-generator/src/test/resources/positive/BidirectionalComputation.java new file mode 100644 index 00000000000..1d84ceefe2c --- /dev/null +++ b/pregel-proc-generator/src/test/resources/positive/BidirectionalComputation.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.beta.pregel.cc; + +import org.neo4j.gds.beta.pregel.BidirectionalPregelComputation; +import org.neo4j.gds.beta.pregel.Messages; +import org.neo4j.gds.beta.pregel.PregelComputation; +import org.neo4j.gds.beta.pregel.PregelProcedureConfig; +import org.neo4j.gds.beta.pregel.context.ComputeContext; +import org.neo4j.gds.beta.pregel.PregelSchema; +import org.neo4j.gds.beta.pregel.annotation.GDSMode; +import org.neo4j.gds.beta.pregel.annotation.PregelProcedure; +import org.neo4j.gds.beta.pregel.context.ComputeContext.BidirectionalComputeContext; + +@PregelProcedure( + name = "gds.pregel.bidirectionalTest", + description = "Bidirectional Test computation description", + modes = {GDSMode.STREAM} +) +public class BidirectionalComputation implements BidirectionalPregelComputation { + + @Override + public PregelSchema schema(PregelProcedureConfig config) { + return null; + } + + @Override + public void compute(BidirectionalComputeContext context, Messages messages) { + + } +} diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java index fd8b35d720c..87d107d451d 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java @@ -43,8 +43,9 @@ public final class ForkJoinComputeStep< private final InitFunction initFunction; private final ComputeFunction computeFunction; - private final Supplier initContext; - private final Supplier computeContext; + private final Supplier initContextSupplier; + private final Supplier computeContextSupplier; + private final COMPUTE_CONTEXT computeContext; private final NodeValue nodeValue; private final HugeAtomicBitSet voteBits; private final Messenger messenger; @@ -56,8 +57,8 @@ public final class ForkJoinComputeStep< ForkJoinComputeStep( InitFunction initFunction, ComputeFunction computeFunction, - Supplier initContext, - Supplier computeContext, + Supplier initContextSupplier, + Supplier computeContextSupplier, MutableInt iteration, Partition nodeBatch, NodeValue nodeValue, @@ -70,8 +71,8 @@ public final class ForkJoinComputeStep< super(parent); this.initFunction = initFunction; this.computeFunction = computeFunction; - this.initContext = initContext; - this.computeContext = computeContext; + this.initContextSupplier = initContextSupplier; + this.computeContextSupplier = computeContextSupplier; this.iteration = iteration; this.voteBits = voteBits; this.nodeBatch = nodeBatch; @@ -79,6 +80,7 @@ public final class ForkJoinComputeStep< this.messenger = messenger; this.hasSentMessage = sentMessage; this.progressTracker = progressTracker; + this.computeContext = computeContextSupplier.get(); } @Override @@ -99,8 +101,8 @@ public void compute() { var leftTask = new ForkJoinComputeStep<>( initFunction, computeFunction, - initContext, - computeContext, + initContextSupplier, + computeContextSupplier, iteration, leftBatch, nodeValue, @@ -119,6 +121,7 @@ public void compute() { this.compute(); } else { computeBatch(); + hasSentMessage.compareAndSet(false, computeContext.hasSentMessage()); tryComplete(); } } @@ -155,12 +158,12 @@ public Partition nodeBatch() { @Override public INIT_CONTEXT initContext() { - return initContext.get(); + return initContextSupplier.get(); } @Override public COMPUTE_CONTEXT computeContext() { - return computeContext.get(); + return computeContext; } @Override diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java index 6f480a62300..096c40e2953 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java @@ -30,6 +30,7 @@ import org.neo4j.gds.core.utils.partition.Partition; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import java.util.Optional; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; @@ -103,12 +104,12 @@ void release() { Supplier> computeContext = () -> new ComputeContext<>( graph.concurrentCopy(), config, - ((PregelComputation) computation)::applyRelationshipWeight, + computation, nodeValues, messenger, voteBits, iteration, - hasSentMessages, + Optional.empty(), progressTracker ); @@ -144,12 +145,12 @@ void release() { Supplier> computeContext = () -> new BidirectionalComputeContext<>( graph.concurrentCopy(), config, - ((BidirectionalPregelComputation) computation)::applyRelationshipWeight, + computation, nodeValues, messenger, voteBits, iteration, - hasSentMessages, + Optional.empty(), progressTracker ); diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java index e8b242d2ab5..8e195f4fa42 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.beta.pregel; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.beta.pregel.context.ComputeContext; import org.neo4j.gds.beta.pregel.context.InitContext; @@ -26,8 +27,6 @@ import org.neo4j.gds.core.utils.partition.Partition; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; -import java.util.concurrent.atomic.AtomicBoolean; - public final class PartitionedComputeStep< CONFIG extends PregelConfig, ITERATOR extends Messages.MessageIterator, @@ -45,7 +44,7 @@ public final class PartitionedComputeStep< private final Messenger messenger; private final MutableInt iteration; - private final AtomicBoolean hasSentMessage; + private final MutableBoolean hasSentMessage; private final NodeValue nodeValue; PartitionedComputeStep( @@ -58,7 +57,7 @@ public final class PartitionedComputeStep< Messenger messenger, HugeAtomicBitSet voteBits, MutableInt iteration, - AtomicBoolean hasSentMessage, + MutableBoolean hasSentMessage, ProgressTracker progressTracker ) { this.initFunction = initFunction; @@ -127,10 +126,10 @@ public ProgressTracker progressTracker() { void init(int iteration) { this.iteration.setValue(iteration); - this.hasSentMessage.set(false); + hasSentMessage.setValue(false); } boolean hasSentMessage() { - return hasSentMessage.get(); + return hasSentMessage.getValue(); } } diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java index e5285a8e2b4..c0b0da73cb2 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.beta.pregel; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableInt; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.api.Graph; @@ -35,7 +36,6 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -137,7 +137,7 @@ void release() { Partition partition ) { MutableInt iteration = new MutableInt(0); - var hasSentMessages = new AtomicBoolean(false); + var hasSentMessages = new MutableBoolean(false); var initContext = new InitContext<>( graph, @@ -149,12 +149,12 @@ void release() { var computeContext = new ComputeContext<>( graph, config, - ((PregelComputation) computation)::applyRelationshipWeight, + computation, nodeValues, messenger, voteBits, iteration, - hasSentMessages, + Optional.of(hasSentMessages), progressTracker ); @@ -180,7 +180,7 @@ void release() { Partition partition ) { MutableInt iteration = new MutableInt(0); - var hasSentMessages = new AtomicBoolean(false); + var hasSentMessages = new MutableBoolean(false); var initContext = new BidirectionalInitContext<>( graph, @@ -192,12 +192,12 @@ void release() { var computeContext = new BidirectionalComputeContext<>( graph, config, - ((BidirectionalPregelComputation) computation)::applyRelationshipWeight, + computation, nodeValues, messenger, voteBits, iteration, - hasSentMessages, + Optional.of(hasSentMessages), progressTracker ); diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java index 380166b9af3..57064361d92 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java @@ -19,15 +19,17 @@ */ package org.neo4j.gds.beta.pregel.context; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.beta.pregel.BasePregelComputation; import org.neo4j.gds.beta.pregel.Messenger; import org.neo4j.gds.beta.pregel.NodeValue; import org.neo4j.gds.beta.pregel.PregelConfig; import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.Optional; /** * A context that is used during the computation. It allows an implementation @@ -36,31 +38,32 @@ */ public class ComputeContext extends NodeCentricContext { - final RelationshipWeightApplier relationshipWeightApplier; private final HugeAtomicBitSet voteBits; private final Messenger messenger; private final MutableInt iteration; - private final AtomicBoolean hasSendMessage; + private final MutableBoolean hasSendMessage; + + protected BasePregelComputation computation; public ComputeContext(Graph graph, CONFIG config, - RelationshipWeightApplier relationshipWeightApplier, + BasePregelComputation computation, NodeValue nodeValue, Messenger messenger, HugeAtomicBitSet voteBits, MutableInt iteration, - AtomicBoolean hasSendMessage, + Optional hasSendMessage, ProgressTracker progressTracker) { super(graph, config, nodeValue, progressTracker); - this.relationshipWeightApplier = relationshipWeightApplier; + this.computation = computation; this.sendMessagesFunction = config.hasRelationshipWeightProperty() ? this::sendToNeighborsWeighted : this::sendToNeighbors; this.messenger = messenger; this.voteBits = voteBits; this.iteration = iteration; - this.hasSendMessage = hasSendMessage; + this.hasSendMessage = hasSendMessage.orElse(new MutableBoolean(false)); } private final SendMessagesFunction sendMessagesFunction; @@ -151,7 +154,7 @@ public void sendToNeighbors(double message) { */ public void sendTo(long targetNodeId, double message) { messenger.sendTo(targetNodeId, message); - this.hasSendMessage.set(true); + this.hasSendMessage.setValue(true); } private void sendToNeighbors(long sourceNodeId, double message) { @@ -163,19 +166,18 @@ private void sendToNeighbors(long sourceNodeId, double message) { private void sendToNeighborsWeighted(long sourceNodeId, double message) { graph.forEachRelationship(sourceNodeId, 1.0, (ignored, targetNodeId, weight) -> { - sendTo(targetNodeId, relationshipWeightApplier.applyRelationshipWeight(message, weight)); + sendTo(targetNodeId, computation.applyRelationshipWeight(message, weight)); return true; }); } - @FunctionalInterface - interface SendMessagesFunction { - void sendToNeighbors(long sourceNodeId, double message); + public boolean hasSentMessage() { + return hasSendMessage.getValue(); } @FunctionalInterface - public interface RelationshipWeightApplier { - double applyRelationshipWeight(double nodeValue, double relationshipWeight); + interface SendMessagesFunction { + void sendToNeighbors(long sourceNodeId, double message); } public static final class BidirectionalComputeContext extends ComputeContext implements BidirectionalNodeCentricContext { @@ -185,18 +187,18 @@ public static final class BidirectionalComputeContext computation, NodeValue nodeValue, Messenger messenger, HugeAtomicBitSet voteBits, MutableInt iteration, - AtomicBoolean hasSendMessage, + Optional hasSendMessage, ProgressTracker progressTracker ) { super( graph, config, - relationshipWeightApplier, + computation, nodeValue, messenger, voteBits, @@ -226,7 +228,7 @@ private void sendToIncomingNeighbors(long sourceNodeId, double message) { private void sendToIncomingNeighborsWeighted(long sourceNodeId, double message) { graph.forEachInverseRelationship(sourceNodeId, 1.0, (ignored, targetNodeId, weight) -> { - sendTo(targetNodeId, relationshipWeightApplier.applyRelationshipWeight(message, weight)); + sendTo(targetNodeId, computation.applyRelationshipWeight(message, weight)); return true; }); } diff --git a/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java b/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java index c155cd16a49..0dbd8f0c4a9 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java +++ b/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java @@ -35,13 +35,13 @@ final class Node2VecCompanion { static NodePropertyValues nodeProperties( ComputationResult computationResult ) { - var size = computationResult.graph().nodeCount(); + var nodeCount = computationResult.graph().nodeCount(); HugeObjectArray embeddings = computationResult.result().embeddings(); return new FloatArrayNodePropertyValues() { @Override - public long size() { - return size; + public long nodeCount() { + return nodeCount; } @Override diff --git a/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java b/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java index 5d721bf6cc6..f30d9541520 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java +++ b/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java @@ -65,7 +65,9 @@ protected AbstractResultBuilder resultBuilder( ComputationResult computeResult, ExecutionContext executionContext ) { - return new MutateResult.Builder().withInputRelationships(computeResult.graph().relationshipCount()); + return new MutateResult.Builder().withInputRelationships(computeResult + .graphStore() + .relationshipCount(RelationshipType.of(computeResult.config().relationshipType()))); } @Override @@ -77,10 +79,7 @@ protected void updateGraphStore( ComputationResult computationResult, ExecutionContext executionContext ) { - computationResult.graphStore().addRelationshipType( - RelationshipType.of(computationResult.config().mutateRelationshipType()), - computationResult.result() - ); + computationResult.graphStore().addRelationshipType(computationResult.result()); resultBuilder.withRelationshipsWritten(computationResult.result().topology().elementCount()); } }; diff --git a/proc/beta/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationProc.java b/proc/beta/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationProc.java index 5b9e3d73f0f..89d442aa43c 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationProc.java +++ b/proc/beta/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationProc.java @@ -50,7 +50,7 @@ static NodePropertyValues nodeProp ) { LongNodePropertyValues resultCommunities = computationResult.result().asNodeProperties(); if (computationResult.config().consecutiveIds()) { - return new ConsecutiveLongNodePropertyValues(resultCommunities, computationResult.graph().nodeCount()); + return new ConsecutiveLongNodePropertyValues(resultCommunities); } else { return resultCommunities; } diff --git a/proc/beta/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java b/proc/beta/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java index 0cdfbbeec4b..6d390e2dc1a 100644 --- a/proc/beta/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java +++ b/proc/beta/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java @@ -23,23 +23,22 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.AlgoBaseProc; import org.neo4j.gds.GdsCypher; -import org.neo4j.gds.ImmutableRelationshipProjections; import org.neo4j.gds.MutateNodePropertyTest; import org.neo4j.gds.NodeProjections; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; +import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.config.GraphProjectFromStoreConfig; -import org.neo4j.gds.config.ImmutableGraphProjectFromStoreConfig; +import org.neo4j.gds.config.GraphProjectFromStoreConfigImpl; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.graphdb.GraphDatabaseService; import java.util.Arrays; -import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -206,21 +205,20 @@ GdsCypher.ModeBuildStage explicitAlgoBuildStage() { } static String graphProjectQuery() { - GraphProjectFromStoreConfig config = ImmutableGraphProjectFromStoreConfig + GraphProjectFromStoreConfig config = GraphProjectFromStoreConfigImpl .builder() .graphName("") - .nodeProjections(NodeProjections.of()) + .username("") + .nodeProjections(NodeProjections.all()) .nodeProperties(PropertyMappings.fromObject(Arrays.asList("seed1", "seed2"))) - .relationshipProjections(ImmutableRelationshipProjections.builder() - .putProjection( + .relationshipProjections(RelationshipProjections.single( ALL_RELATIONSHIPS, RelationshipProjection.builder() .type("TYPE") .orientation(Orientation.UNDIRECTED) - .properties(PropertyMappings.of( - Collections.singletonList(PropertyMapping.of("weight", 1D)))) + .addProperty(PropertyMapping.of("weight", 1D)) .build() - ).build() + ) ).build(); return GdsCypher diff --git a/proc/catalog/src/main/java/org/neo4j/gds/beta/generator/GraphGenerateProc.java b/proc/catalog/src/main/java/org/neo4j/gds/beta/generator/GraphGenerateProc.java index 225f8e6a42f..c8b56c6f717 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/beta/generator/GraphGenerateProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/beta/generator/GraphGenerateProc.java @@ -98,7 +98,6 @@ private GraphGenerationStats generateGraph( GraphStore graphStore = CSRGraphStoreUtil.createFromGraph( DatabaseId.of(databaseService), graph, - config.relationshipType().name, relationshipProperty, config.readConcurrency() ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java index b2b99e2aa3e..05770540154 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java @@ -57,7 +57,7 @@ public Stream run( config.validate(graphStore); // removing - long propertiesRemoved = graphStore.graphPropertyValues(graphProperty).size(); + long propertiesRemoved = graphStore.graphPropertyValues(graphProperty).valueCount(); runWithExceptionLogging( "Graph property removal failed", () -> graphStore.removeGraphProperty(graphProperty) diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java index cfbef0cb63a..4d4210ff286 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java @@ -120,7 +120,7 @@ private Long dropNodeProperties( progressTracker.beginSubTask(); config.nodeProperties().forEach(propertyKey -> { - removedPropertiesCount.add(graphStore.nodeProperty(propertyKey).values().size()); + removedPropertiesCount.add(graphStore.nodeProperty(propertyKey).values().nodeCount()); graphStore.removeNodeProperty(propertyKey); progressTracker.logProgress(); }); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProc.java index 1f1f850c535..ab8cd73077a 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProc.java @@ -22,7 +22,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.ProcPreconditions; import org.neo4j.gds.beta.filter.NodesFilter; -import org.neo4j.gds.beta.filter.expression.ExpressionParser; import org.neo4j.gds.config.MutateLabelConfig; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.ProgressTimer; @@ -36,6 +35,7 @@ import java.util.concurrent.atomic.LongAdder; import java.util.stream.Stream; +import static org.neo4j.gds.catalog.NodeFilterParser.parseAndValidate; import static org.neo4j.procedure.Mode.READ; public class GraphMutateNodeLabelProc extends CatalogProc { @@ -52,14 +52,14 @@ public Stream mutate( var procedureConfig = MutateLabelConfig.of(configuration); var graphStore = graphStoreFromCatalog(graphName).graphStore(); - var filter = ExpressionParser.parse(procedureConfig.nodeFilter(), Map.of()); + var nodeFilter = parseAndValidate(graphStore, procedureConfig.nodeFilter()); var nodeLabelToMutate = NodeLabel.of(nodeLabel); var resultBuilder = MutateLabelResult.builder(graphName, nodeLabel).withConfig(procedureConfig.toMap()); try (ProgressTimer ignored = ProgressTimer.start(resultBuilder::withMutateMillis)) { var filteredNodes = NodesFilter.filterNodes( graphStore, - filter, + nodeFilter, procedureConfig.concurrency(), Map.of(), Pools.DEFAULT, @@ -87,5 +87,4 @@ public Stream mutate( return Stream.of(resultBuilder.build()); } - } diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java index fcfd27fcc7c..6aa25dcb9a5 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java @@ -21,7 +21,6 @@ import org.neo4j.gds.ProcPreconditions; import org.neo4j.gds.beta.filter.NodesFilter; -import org.neo4j.gds.beta.filter.expression.ExpressionParser; import org.neo4j.gds.config.WriteLabelConfig; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.ProgressTimer; @@ -38,6 +37,7 @@ import java.util.Map; import java.util.stream.Stream; +import static org.neo4j.gds.catalog.NodeFilterParser.parseAndValidate; import static org.neo4j.procedure.Mode.WRITE; public class GraphWriteNodeLabelProc extends CatalogProc { @@ -56,16 +56,15 @@ public Stream write( ProcPreconditions.check(); var procedureConfig = WriteLabelConfig.of(configuration); - var graphStore = graphStoreFromCatalog(graphName).graphStore(); - var filter = ExpressionParser.parse(procedureConfig.nodeFilter(), Map.of()); + var nodeFilter = parseAndValidate(graphStore, procedureConfig.nodeFilter()); var resultBuilder = WriteLabelResult.builder(graphName, nodeLabel) .withConfig(procedureConfig.toMap()); try (ProgressTimer ignored = ProgressTimer.start(resultBuilder::withWriteMillis)) { var filteredNodes = NodesFilter.filterNodes( graphStore, - filter, + nodeFilter, procedureConfig.concurrency(), Map.of(), Pools.DEFAULT, diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeFilterParser.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeFilterParser.java new file mode 100644 index 00000000000..ade93552021 --- /dev/null +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeFilterParser.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.catalog; + +import org.neo4j.gds.ElementProjection; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.beta.filter.expression.Expression; +import org.neo4j.gds.beta.filter.expression.ExpressionParser; +import org.neo4j.gds.beta.filter.expression.SemanticErrors; +import org.neo4j.gds.beta.filter.expression.ValidationContext; +import org.opencypher.v9_0.parser.javacc.ParseException; + +final class NodeFilterParser { + + private NodeFilterParser() {} + + static Expression parseAndValidate(GraphStore graphStore, String nodeFilter) throws IllegalArgumentException { + try { + var validationContext = ValidationContext.forNodes(graphStore); + var filter = ExpressionParser.parse( + nodeFilter.equals(ElementProjection.PROJECT_ALL) ? "true" : nodeFilter, + validationContext.availableProperties() + ); + filter.validate(validationContext).validate(); + return filter; + } catch (ParseException | SemanticErrors e) { + throw new IllegalArgumentException("Invalid `nodeFilter` expression.", e); + } + } + +} diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java index b9b34fd52bc..65140877e6e 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java @@ -68,9 +68,8 @@ public LongStream longValues() { } @Override - public long size() { + public long valueCount() { return 10; - } }; @@ -113,9 +112,8 @@ public LongStream longValues() { } @Override - public long size() { + public long valueCount() { return 10; - } }; graphStore.addGraphProperty("prop", values); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProcTest.java index 6ad4639490b..753ae9c0bf3 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProcTest.java @@ -27,7 +27,6 @@ import org.neo4j.gds.GdsCypher; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.core.loading.CatalogRequest; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.warnings.GlobalUserLogStore; @@ -90,19 +89,19 @@ void setup() throws Exception { runQuery(GdsCypher.call(TEST_GRAPH_DIFFERENT_PROPERTIES) .graphProject() - .withNodeLabel("A", NodeProjection.of( - "A", - PropertyMappings.of().withMappings( + .withNodeLabel("A", NodeProjection.builder() + .label("A") + .addProperties( PropertyMapping.of("nodeProp1", 1337), PropertyMapping.of("nodeProp2", 1337) - ) - )) - .withNodeLabel("B", NodeProjection.of( - "B", - PropertyMappings.of().withMappings( + ).build() + ) + .withNodeLabel("B", NodeProjection.builder() + .label("B") + .addProperty( PropertyMapping.of("nodeProp1", 1337) - ) - )) + ).build() + ) .withAnyRelationshipType() .yields() ); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropProcTest.java index 3a09f3d0acc..390b6f7d970 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropProcTest.java @@ -94,7 +94,7 @@ void dropGraphFromCatalog() { new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(9) + .hasSize(10) .containsEntry( "nodeProjection", map( "A", map( @@ -126,6 +126,7 @@ void dropGraphFromCatalog() { intAssertConsumer(readConcurrency -> readConcurrency.isEqualTo(4)) ) .hasEntrySatisfying("sudo", booleanAssertConsumer(AbstractBooleanAssert::isFalse)) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) .doesNotContainKeys( "username", GraphProjectConfig.NODE_COUNT_KEY, diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphInfoTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphInfoTest.java index 7c70e89d66a..788c458cdb9 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphInfoTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphInfoTest.java @@ -102,7 +102,6 @@ private GraphInfo create( var graphStore = CSRGraphStoreUtil.createFromGraph( DatabaseId.from("test"), graph, - "TY", Optional.empty(), 1 ); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListProcTest.java index 3d181f9c95c..f6e2524cfaa 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListProcTest.java @@ -104,7 +104,7 @@ void listASingleLabelRelationshipTypeProjection() { new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(9) + .hasSize(10) .containsEntry( "nodeProjection", map( "A", map( @@ -136,6 +136,7 @@ void listASingleLabelRelationshipTypeProjection() { intAssertConsumer(readConcurrency -> readConcurrency.isEqualTo(4)) ) .hasEntrySatisfying("sudo", booleanAssertConsumer(AbstractBooleanAssert::isFalse)) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) .doesNotContainKeys( GraphProjectConfig.NODE_COUNT_KEY, GraphProjectConfig.RELATIONSHIP_COUNT_KEY, @@ -217,7 +218,7 @@ void listGeneratedGraph() { "configuration", new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(11) + .hasSize(12) .containsEntry("nodeProjections", map( "10_Nodes", map( "label", "10_Nodes", @@ -250,6 +251,7 @@ void listGeneratedGraph() { ) .hasEntrySatisfying("allowSelfLoops", booleanAssertConsumer(AbstractBooleanAssert::isFalse)) .hasEntrySatisfying("sudo", booleanAssertConsumer(AbstractBooleanAssert::isFalse)) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) .hasEntrySatisfying( "relationshipDistribution", stringAssertConsumer(relationshipDistribution -> relationshipDistribution.isEqualTo( @@ -344,7 +346,7 @@ void listCypherProjection() { "configuration", new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(8) + .hasSize(9) .hasEntrySatisfying( "relationshipQuery", stringAssertConsumer(relationshipQuery -> relationshipQuery.isEqualTo( @@ -360,6 +362,7 @@ void listCypherProjection() { stringAssertConsumer(nodeQuery -> nodeQuery.isEqualTo(ALL_NODES_QUERY)) ) .hasEntrySatisfying("sudo", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) .hasEntrySatisfying( "readConcurrency", intAssertConsumer(readConcurrency -> readConcurrency.isEqualTo(4)) @@ -417,11 +420,12 @@ void listCypherAggregation() { "configuration", new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(4) + .hasSize(5) .hasEntrySatisfying("creationTime", creationTimeAssertConsumer()) .hasEntrySatisfying("jobId", jobId -> assertThat(jobId).isNotNull()) .hasEntrySatisfying("undirectedRelationshipTypes", t -> assertThat(t).isEqualTo(List.of())) - .hasEntrySatisfying("inverseIndexedRelationshipTypes", t -> assertThat(t).isEqualTo(List.of())); + .hasEntrySatisfying("inverseIndexedRelationshipTypes", t -> assertThat(t).isEqualTo(List.of())) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)); return true; }, "Assert Cypher Aggregation `configuration` map"), diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProcTest.java index c6e47657086..8c8e9a55ba2 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProcTest.java @@ -19,27 +19,37 @@ */ package org.neo4j.gds.catalog; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.graphdb.QueryExecutionException; +import java.util.Map; import java.util.concurrent.atomic.LongAdder; -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.InstanceOfAssertFactories.DOUBLE; import static org.assertj.core.api.InstanceOfAssertFactories.LONG; +@ExtendWith(SoftAssertionsExtension.class) class GraphMutateNodeLabelProcTest extends BaseProcTest { @Neo4jGraph(offsetIds = true) private static final String DB_CYPHER = "CREATE" + - " (a:A { p: 1.0 })" + - ", (b:A { p: 2.0 })" + - ", (c:A { p: 3.0 })" + + " (a:A {longProperty: 1, floatProperty: 15.5})" + + ", (b:A {longProperty: 2, floatProperty: 23})" + + ", (c:A {longProperty: 3, floatProperty: 18.3})" + ", (d:B)"; @Inject @@ -56,12 +66,12 @@ void setUp() throws Exception { } @Test - void mutateNodeLabelMultiLabelProjection() { + void mutateNodeLabelMultiLabelProjection(SoftAssertions assertions) { // Arrange runQuery( "CALL gds.graph.project('graph', " + "{" + - " A: { properties: 'p' }," + + " A: { properties: 'longProperty' }," + " B: { label: 'B' }" + "}, " + "'*')" @@ -69,24 +79,24 @@ void mutateNodeLabelMultiLabelProjection() { // Act runQuery( - "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n:A AND n.p > 1.0' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n:A AND n.longProperty > 1' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", result -> { - assertThat(result.hasNext()).isTrue(); + assertions.assertThat(result.hasNext()).isTrue(); var row = result.next(); - assertThat(row.get("nodeCount")) + assertions.assertThat(row.get("nodeCount")) .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") .isEqualTo(4L); - assertThat(row.get("nodeLabel")) + assertions.assertThat(row.get("nodeLabel")) .as("The specified node label should be present in the result") .isEqualTo("TestLabel"); - assertThat(row.get("nodeLabelsWritten")) + assertions.assertThat(row.get("nodeLabelsWritten")) .as("There should be two nodes having the new label in the in-memory graph") .isEqualTo(2L); - assertThat(result.hasNext()).isFalse(); + assertions.assertThat(result.hasNext()).isFalse(); return false; } ); @@ -94,9 +104,9 @@ void mutateNodeLabelMultiLabelProjection() { // Assert var counter = new LongAdder(); runQueryWithRowConsumer( - "CALL gds.graph.nodeProperty.stream('graph', 'p', ['TestLabel'])", + "CALL gds.graph.nodeProperty.stream('graph', 'longProperty', ['TestLabel'])", row -> { - assertThat(row.getNumber("nodeId")) + assertions.assertThat(row.getNumber("nodeId")) .asInstanceOf(LONG) .as("Only nodes `b` and `c` should have the `TestLabel` applied.") .isIn( @@ -104,45 +114,45 @@ void mutateNodeLabelMultiLabelProjection() { idFunction.of("c") ); - assertThat(row.getNumber("propertyValue")) - .asInstanceOf(DOUBLE) + assertions.assertThat(row.getNumber("propertyValue")) + .asInstanceOf(LONG) .as("The node property should match the applied filter") - .isGreaterThan(1d); + .isGreaterThan(1); counter.increment(); } ); - assertThat(counter.intValue()) + assertions.assertThat(counter.intValue()) .as("There should have been two steamed nodes") .isEqualTo(2); } @Test - void mutateNodeLabelSingleLabelProjection() { + void mutateNodeLabelSingleLabelProjection(SoftAssertions assertions) { // Arrange runQuery( "CALL gds.graph.project('graph', " + "{" + - " A: { properties: 'p' }" + + " A: { properties: 'longProperty' }" + "}, " + "'*')" ); // Act runQuery( - "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n:A AND n.p > 1.0' }) YIELD nodeCount, nodeLabelsWritten", + "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n:A AND n.longProperty > 1' }) YIELD nodeCount, nodeLabelsWritten", result -> { - assertThat(result.hasNext()).isTrue(); + assertions.assertThat(result.hasNext()).isTrue(); var row = result.next(); - assertThat(row.get("nodeCount")) + assertions.assertThat(row.get("nodeCount")) .as("Total number of nodes in the graph should be three, including the nodes that didn't get the new label") .isEqualTo(3L); - assertThat(row.get("nodeLabelsWritten")) + assertions.assertThat(row.get("nodeLabelsWritten")) .as("There should be two nodes having the new label in the in-memory graph") .isEqualTo(2L); - assertThat(result.hasNext()).isFalse(); + assertions.assertThat(result.hasNext()).isFalse(); return false; } ); @@ -150,9 +160,9 @@ void mutateNodeLabelSingleLabelProjection() { // Assert var counter = new LongAdder(); runQueryWithRowConsumer( - "CALL gds.graph.nodeProperty.stream('graph', 'p', ['TestLabel'])", + "CALL gds.graph.nodeProperty.stream('graph', 'longProperty', ['TestLabel'])", row -> { - assertThat(row.getNumber("nodeId")) + assertions.assertThat(row.getNumber("nodeId")) .asInstanceOf(LONG) .as("Only nodes `b` and `c` should have the `TestLabel` applied.") .isIn( @@ -160,47 +170,105 @@ void mutateNodeLabelSingleLabelProjection() { idFunction.of("c") ); - assertThat(row.getNumber("propertyValue")) - .asInstanceOf(DOUBLE) + assertions.assertThat(row.getNumber("propertyValue")) + .asInstanceOf(LONG) .as("The node property should match the applied filter") - .isGreaterThan(1d); + .isGreaterThan(1); counter.increment(); } ); - assertThat(counter.intValue()) + assertions.assertThat(counter.intValue()) .as("There should have been two steamed nodes") .isEqualTo(2); } @Test - void mutateNodeLabelStarProjection() { + void mutateNodeLabelStarProjection(SoftAssertions assertions) { // Arrange runQuery( "CALL gds.graph.project('graph', " + "'*'," + "'*'," + - "{ nodeProperties: {p: {defaultValue: 0.0}}}" + + "{nodeProperties: {longProperty: {defaultValue: 0}}}" + ")" ); // Act runQuery( - "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n.p > 2.0' }) YIELD nodeCount, nodeLabelsWritten", + "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n.longProperty <= 2' }) YIELD nodeCount, nodeLabelsWritten", + result -> { + assertions.assertThat(result.hasNext()).isTrue(); + + var row = result.next(); + assertions.assertThat(row.get("nodeCount")) + .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") + .isEqualTo(4L); + + assertions.assertThat(row.get("nodeLabelsWritten")) + .as("There should be three nodes having the new label in the in-memory graph") + .isEqualTo(3L); + + assertions.assertThat(result.hasNext()).isFalse(); + return false; + } + ); + + // Assert + var counter = new LongAdder(); + runQueryWithRowConsumer( + "CALL gds.graph.nodeProperty.stream('graph', 'longProperty', ['TestLabel'])", + row -> { + assertions.assertThat(row.getNumber("nodeId")) + .asInstanceOf(LONG) + .as("Nodes `a` and `b` and `d` should have the `TestLabel` applied.") + .isIn( + idFunction.of("a"), + idFunction.of("b"), + idFunction.of("d") + ); + + assertions.assertThat(row.getNumber("propertyValue")) + .asInstanceOf(LONG) + .as("The node property should match the applied filter") + .isLessThanOrEqualTo(2); + + counter.increment(); + } + ); + assertions.assertThat(counter.intValue()) + .as("There should have been three steamed nodes") + .isEqualTo(3); + } + + @Test + void shouldWorkWithFloatProperties(SoftAssertions assertions) { + // Arrange + runQuery("CALL gds.graph.project('graph', " + + "{" + + " A: { properties: 'floatProperty' }," + + " B: { label: 'B' }" + + "}, " + + "'*')"); + + + // Act + runQuery( + "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n.floatProperty <= 19.0' }) YIELD nodeCount, nodeLabelsWritten", result -> { - assertThat(result.hasNext()).isTrue(); + assertions.assertThat(result.hasNext()).isTrue(); var row = result.next(); - assertThat(row.get("nodeCount")) + assertions.assertThat(row.get("nodeCount")) .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") .isEqualTo(4L); - assertThat(row.get("nodeLabelsWritten")) + assertions.assertThat(row.get("nodeLabelsWritten")) .as("There should be two nodes having the new label in the in-memory graph") - .isEqualTo(1L); + .isEqualTo(2L); - assertThat(result.hasNext()).isFalse(); + assertions.assertThat(result.hasNext()).isFalse(); return false; } ); @@ -208,26 +276,56 @@ void mutateNodeLabelStarProjection() { // Assert var counter = new LongAdder(); runQueryWithRowConsumer( - "CALL gds.graph.nodeProperty.stream('graph', 'p', ['TestLabel'])", + "CALL gds.graph.nodeProperty.stream('graph', 'floatProperty', ['TestLabel'])", row -> { - assertThat(row.getNumber("nodeId")) + assertions.assertThat(row.getNumber("nodeId")) .asInstanceOf(LONG) - .as("Only node `c` should have the `TestLabel` applied.") - .isEqualTo( + .as("Nodes `a` and `c` should have the `TestLabel` applied.") + .isIn( + idFunction.of("a"), idFunction.of("c") ); - assertThat(row.getNumber("propertyValue")) + assertions.assertThat(row.getNumber("propertyValue")) .asInstanceOf(DOUBLE) .as("The node property should match the applied filter") - .isGreaterThan(2d); + .isLessThanOrEqualTo(19d); counter.increment(); } ); - assertThat(counter.intValue()) - .as("There should have been one steamed node") - .isEqualTo(1); + assertions.assertThat(counter.intValue()) + .as("There should have been two steamed nodes") + .isEqualTo(2); + } + + // Only check two scenarios, the rest are covered in NodeFilterParserTest. + @ParameterizedTest + @ValueSource( + strings = { + "n.floatProperty > 10", + "n.longProperty <= 19.6", + } + ) + void shouldFailOnIncompatiblePropertyAndValue(String nodeFilter) { + runQuery( + "CALL gds.graph.project('graph', " + + "{" + + " A: { properties: ['longProperty', 'floatProperty'] }," + + " B: { label: 'B' }" + + "}, " + + "'*')" + ); + + assertThatExceptionOfType(QueryExecutionException.class) + .isThrownBy(() -> runQuery("CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: $nodeFilter })", + Map.of("nodeFilter", nodeFilter))) + .withMessageContaining("Semantic errors while parsing expression") + .withMessageContaining("Incompatible types"); + } + @AfterEach + void tearDown() { + GraphStoreCatalog.removeAllLoadedGraphs(); } } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java index c86804cd3ff..040467e68e9 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java @@ -82,8 +82,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.neo4j.gds.AbstractNodeProjection.LABEL_KEY; import static org.neo4j.gds.ElementProjection.PROPERTIES_KEY; +import static org.neo4j.gds.NodeProjection.LABEL_KEY; import static org.neo4j.gds.RelationshipProjection.AGGREGATION_KEY; import static org.neo4j.gds.RelationshipProjection.INDEX_INVERSE_KEY; import static org.neo4j.gds.RelationshipProjection.ORIENTATION_KEY; diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java index fd61d1afb38..d1277f20d47 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java @@ -344,7 +344,7 @@ void failsDatabaseExportWhenRunningOnCluster() { .hasMessageContaining("The requested operation") .hasMessageContaining("(Export a graph to Neo4j database)") .hasMessageContaining( - "is not available while running Neo4j Graph Data Science library on a Neo4j Causal Cluster." + "is not available while running Neo4j Graph Data Science library on a Neo4j Cluster." ); } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java index b417244e63b..a4c67dc1555 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java @@ -92,9 +92,8 @@ public LongStream longValues() { } @Override - public long size() { + public long valueCount() { return 10; - } }), arguments(new DoubleGraphPropertyValues() { @Override @@ -103,7 +102,7 @@ public DoubleStream doubleValues() { } @Override - public long size() { + public long valueCount() { return 3; } }), arguments(new LongArrayGraphPropertyValues() { @@ -113,7 +112,7 @@ public Stream longArrayValues() { } @Override - public long size() { + public long valueCount() { return 2; } })); @@ -133,13 +132,12 @@ void shouldFailOnNonExistingNodePropertyForSpecificLabel() { LongGraphPropertyValues values = new LongGraphPropertyValues() { @Override public LongStream longValues() { - return LongStream.range(0, 10); + return LongStream.range(0, valueCount()); } @Override - public long size() { + public long valueCount() { return 10; - } }; graphStore.addGraphProperty("prop", values); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java index 733869a368b..4c86e3d06ac 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java @@ -31,7 +31,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; @@ -100,19 +99,20 @@ void setup() throws Exception { runQuery(GdsCypher.call(TEST_GRAPH_DIFFERENT_PROPERTIES) .graphProject() - .withNodeLabel("A", NodeProjection.of( - "A", - PropertyMappings.of().withMappings( + .withNodeLabel("A", NodeProjection.builder() + .label("A") + .addProperties( PropertyMapping.of("newNodeProp1", "nodeProp1", 1337), PropertyMapping.of("newNodeProp2", "nodeProp2", 1337) - ) - )) - .withNodeLabel("B", NodeProjection.of( - "B", - PropertyMappings.of().withMappings( - PropertyMapping.of("newNodeProp1", "nodeProp1", 1337) - ) - )) + ).build() + ) + .withNodeLabel("B", + NodeProjection + .builder() + .label("B") + .addProperty(PropertyMapping.of("newNodeProp1", "nodeProp1", 1337)) + .build() + ) .withAnyRelationshipType() .yields() ); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java index 48c0033012b..dcabce64833 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java @@ -207,13 +207,14 @@ void streamMutatedRelationshipProperties() { RelationshipsBuilder relImporter = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(RelationshipType.of("NEW_REL")) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of("newRelProp3")) .build(); relImporter.addFromInternal(0, 1, 23D); - graphStore.addRelationshipType(RelationshipType.of("NEW_REL"), relImporter.build()); + graphStore.addRelationshipType(relImporter.build()); String graphStreamQuery = formatWithLocale( "CALL gds.graph.relationshipProperties.stream(" + @@ -283,17 +284,19 @@ void streamLoadedRelationshipPropertyForTypeSubset() { @Test void streamMutatedNodeProperty() { + GraphStore graphStore = GraphStoreCatalog.get(getUsername(), DatabaseId.of(db), TEST_GRAPH_SAME_PROPERTIES).graphStore(); RelationshipsBuilder relImporter = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(RelationshipType.of("NEW_REL")) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of("newRelProp3")) .build(); relImporter.addFromInternal(0, 1, 23D); - graphStore.addRelationshipType(RelationshipType.of("NEW_REL"), relImporter.build()); + graphStore.addRelationshipType(relImporter.build()); String graphStreamQuery = formatWithLocale( "CALL gds.graph.relationshipProperty.stream(" + diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProcTest.java index 97c3d386af8..eb666460722 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProcTest.java @@ -19,26 +19,35 @@ */ package org.neo4j.gds.catalog; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.BaseProcTest; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.graphdb.QueryExecutionException; +import java.util.Map; import java.util.concurrent.atomic.LongAdder; -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.InstanceOfAssertFactories.DOUBLE; import static org.assertj.core.api.InstanceOfAssertFactories.LONG; +@ExtendWith(SoftAssertionsExtension.class) class GraphWriteNodeLabelProcTest extends BaseProcTest { @Neo4jGraph(offsetIds = true) private static final String DB_CYPHER = "CREATE" + - " (a:A {p: 1})" + - ", (b:A {p: 2})" + - ", (c:A {p: 3})" + + " (a:A {longProperty: 1, floatProperty: 15.5})" + + ", (b:A {longProperty: 2, floatProperty: 23})" + + ", (c:A {longProperty: 3, floatProperty: 18.3})" + ", (d:B)"; @BeforeEach @@ -47,43 +56,44 @@ void setUp() throws Exception { GraphProjectProc.class, GraphWriteNodeLabelProc.class ); - } - @Test - void writeFilteredNodeLabels() { runQuery("CALL gds.graph.project('graph', " + "{" + - " A: { properties: 'p' }," + + " A: { properties: ['longProperty', 'floatProperty'] }," + " B: { label: 'B' }" + "}, " + "'*')"); + } + + @Test + void writeFilteredNodeLabelsWithLongPropertyGreaterComparison(SoftAssertions assertions) { // First make sure the label we want to write doesn't exist runQueryWithRowConsumer( "MATCH (n:TestLabel) RETURN count(n) AS nodeCount", - row -> assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) + row -> assertions.assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) ); runQuery( - "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.p > 1.0' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.longProperty > 1' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", result -> { - assertThat(result.hasNext()).isTrue(); + assertions.assertThat(result.hasNext()).isTrue(); var row = result.next(); - assertThat(row.get("nodeCount")) + assertions.assertThat(row.get("nodeCount")) .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") .isEqualTo(4L); - assertThat(row.get("nodeLabel")) + assertions.assertThat(row.get("nodeLabel")) .as("The specified node label should be present in the result") .isEqualTo("TestLabel"); - assertThat(row.get("nodeLabelsWritten")) + assertions.assertThat(row.get("nodeLabelsWritten")) .as("There should be two nodes having the new label written back to Neo4j") .isEqualTo(2L); - assertThat(result.hasNext()).isFalse(); + assertions.assertThat(result.hasNext()).isFalse(); return false; } ); @@ -91,21 +101,195 @@ void writeFilteredNodeLabels() { // Check that we actually created the labels in the database LongAdder rowCounter = new LongAdder(); runQueryWithRowConsumer( - "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.p AS p", + "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.longProperty AS longProperty", row -> { - assertThat(row.get("updatedLabels")) + assertions.assertThat(row.get("updatedLabels")) .asList() .containsExactlyInAnyOrder("A", "TestLabel"); - assertThat(row.getNumber("p")) + assertions.assertThat(row.getNumber("longProperty")) .asInstanceOf(LONG) .isGreaterThan(1L); rowCounter.increment(); } ); - assertThat(rowCounter.intValue()).isEqualTo(2); + assertions.assertThat(rowCounter.intValue()).isEqualTo(2); + + } + + @Test + void writeFilteredNodeLabelsWithLongPropertyLessThanOrEqualComparison(SoftAssertions assertions) { + // First make sure the label we want to write doesn't exist + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN count(n) AS nodeCount", + row -> assertions.assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) + ); + + runQuery( + "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.longProperty <= 1' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + result -> { + assertions.assertThat(result.hasNext()).isTrue(); + + var row = result.next(); + assertions.assertThat(row.get("nodeCount")) + .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") + .isEqualTo(4L); + + assertions.assertThat(row.get("nodeLabel")) + .as("The specified node label should be present in the result") + .isEqualTo("TestLabel"); + + assertions.assertThat(row.get("nodeLabelsWritten")) + .as("There should be one node having the new label written back to Neo4j") + .isEqualTo(1L); + + + assertions.assertThat(result.hasNext()).isFalse(); + return false; + } + ); + + // Check that we actually created the labels in the database + LongAdder rowCounter = new LongAdder(); + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.longProperty AS longProperty", + row -> { + assertions.assertThat(row.get("updatedLabels")) + .asList() + .containsExactlyInAnyOrder("A", "TestLabel"); + + assertions.assertThat(row.getNumber("longProperty")) + .asInstanceOf(LONG) + .isLessThanOrEqualTo(1L); + + rowCounter.increment(); + } + ); + assertions.assertThat(rowCounter.intValue()) + .as("The MATCH query should return two nodes.") + .isEqualTo(1); + } + + @Test + void writeFilteredNodeLabelsWithFloatPropertyLessThanOrEqualComparison(SoftAssertions assertions) { + // First make sure the label we want to write doesn't exist + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN count(n) AS nodeCount", + row -> assertions.assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) + ); + + runQuery( + "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.floatProperty <= 22.0' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + result -> { + assertions.assertThat(result.hasNext()).isTrue(); + + var row = result.next(); + assertions.assertThat(row.get("nodeCount")) + .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") + .isEqualTo(4L); + + assertions.assertThat(row.get("nodeLabel")) + .as("The specified node label should be present in the result") + .isEqualTo("TestLabel"); + + assertions.assertThat(row.get("nodeLabelsWritten")) + .as("There should be two nodes having the new label written back to Neo4j") + .isEqualTo(2L); + + + assertions.assertThat(result.hasNext()).isFalse(); + return false; + } + ); + + // Check that we actually created the labels in the database + LongAdder rowCounter = new LongAdder(); + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.floatProperty AS floatProperty", + row -> { + assertions.assertThat(row.get("updatedLabels")) + .asList() + .containsExactlyInAnyOrder("A", "TestLabel"); + + assertions.assertThat(row.getNumber("floatProperty")) + .asInstanceOf(DOUBLE) + .isLessThanOrEqualTo(23d); + + rowCounter.increment(); + } + ); + assertions.assertThat(rowCounter.intValue()) + .as("The MATCH query should return two nodes.") + .isEqualTo(2); + + } + + @Test + void writeFilteredNodeLabelsWithFloatPropertyGreaterComparison(SoftAssertions assertions) { + // First make sure the label we want to write doesn't exist + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN count(n) AS nodeCount", + row -> assertions.assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) + ); + + runQuery( + "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.floatProperty > 19.0' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + result -> { + assertions.assertThat(result.hasNext()).isTrue(); + + var row = result.next(); + assertions.assertThat(row.get("nodeCount")) + .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") + .isEqualTo(4L); + + assertions.assertThat(row.get("nodeLabel")) + .as("The specified node label should be present in the result") + .isEqualTo("TestLabel"); + + assertions.assertThat(row.get("nodeLabelsWritten")) + .as("There should be one node having the new label written back to Neo4j") + .isEqualTo(1L); + + + assertions.assertThat(result.hasNext()).isFalse(); + return false; + } + ); + + // Check that we actually created the labels in the database + LongAdder rowCounter = new LongAdder(); + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.floatProperty AS floatProperty", + row -> { + assertions.assertThat(row.get("updatedLabels")) + .asList() + .containsExactlyInAnyOrder("A", "TestLabel"); + + assertions.assertThat(row.getNumber("floatProperty").doubleValue()) + .isGreaterThan(19d); + + rowCounter.increment(); + } + ); + assertions.assertThat(rowCounter.intValue()).isEqualTo(1); + + } + // Only check two scenarios, the rest are covered in NodeFilterParserTest. + @ParameterizedTest + @ValueSource( + strings = { + "n.floatProperty > 10", + "n.longProperty <= 19.6", + } + ) + void shouldFailOnIncompatiblePropertyAndValue(String nodeFilter) { + assertThatExceptionOfType(QueryExecutionException.class) + .isThrownBy(() -> runQuery("CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: $nodeFilter })", + Map.of("nodeFilter", nodeFilter))) + .withMessageContaining("Semantic errors while parsing expression") + .withMessageContaining("Incompatible types"); } @AfterEach diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProcTest.java index d4b20a0b5fa..95b2ab869c8 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProcTest.java @@ -31,7 +31,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; @@ -109,19 +108,21 @@ void setup() throws Exception { runQuery(GdsCypher.call(TEST_GRAPH_DIFFERENT_PROPERTIES) .graphProject() - .withNodeLabel("A", NodeProjection.of( - "A", - PropertyMappings.of().withMappings( + .withNodeLabel("A", NodeProjection.builder() + .label("A") + .addProperties( PropertyMapping.of("newNodeProp1", "nodeProp1", 1337), PropertyMapping.of("newNodeProp2", "nodeProp2", 1337) - ) - )) - .withNodeLabel("B", NodeProjection.of( + ).build() + ) + .withNodeLabel( "B", - PropertyMappings.of().withMappings( - PropertyMapping.of("newNodeProp1", "nodeProp1", 1337) - ) - )) + NodeProjection + .builder() + .label("B") + .addProperty(PropertyMapping.of("newNodeProp1", "nodeProp1", 1337)) + .build() + ) .withAnyRelationshipType() .yields() ); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeFilterParserTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeFilterParserTest.java new file mode 100644 index 00000000000..e3e1b4cc03e --- /dev/null +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeFilterParserTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.catalog; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.beta.filter.expression.SemanticErrors; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.assertThatNoException; + +@GdlExtension +class NodeFilterParserTest { + + @GdlGraph + private static final String DB_CYPHER = + "CREATE" + + " (a:A {longProperty: 1, floatProperty: 15.5})" + + ", (b:A {longProperty: 2, floatProperty: 23})" + + ", (c:A {longProperty: 3, floatProperty: 18.3})" + + ", (d:B)"; + + @Inject + GraphStore graphStore; + + @ParameterizedTest + @ValueSource( + strings = { + "n.floatProperty > 10", + "n.floatProperty >= 11", + "n.floatProperty < 12", + "n.floatProperty <= 19", + "n.floatProperty = 1121", + "n.longProperty > 11.0", + "n.longProperty >= 3.14", + "n.longProperty < 19.6", + "n.longProperty <= 19.6", + "n.longProperty = 21.42", + "n.longProperty <> 2.0", + } + ) + void shouldFailOnIncompatiblePropertyAndValue(String nodeFilter) { + assertThatIllegalArgumentException() + .isThrownBy(() -> NodeFilterParser.parseAndValidate(graphStore, nodeFilter)) + .withRootCauseInstanceOf(SemanticErrors.class) + .havingRootCause() + .withMessageContaining("Semantic errors while parsing expression") + .withMessageContaining("Incompatible types"); + } + + @ParameterizedTest + @ValueSource( + strings = { + "n.longProperty > 10", + "n.longProperty >= 11", + "n.longProperty < 12", + "n.longProperty <= 19", + "n.longProperty = 1121", + "n.floatProperty > 11.0", + "n.floatProperty >= 3.14", + "n.floatProperty < 19.6", + "n.floatProperty <= 19.6", + "n.floatProperty = 21.42", + "n.floatProperty <> 2.0", + } + ) + void shouldNotFailOnCompatiblePropertyAndValue(String nodeFilter) { + assertThatNoException() + .isThrownBy(() -> NodeFilterParser.parseAndValidate(graphStore, nodeFilter)); + } + +} diff --git a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java index 5620d7b0368..7e861aaffc2 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java @@ -51,8 +51,9 @@ static NodePropertyValues nodeProperties(ComputationResult( (computationResult) -> nodePropertyList, (computationResult, executionContext) -> new TestAlgoResultBuilder() @@ -161,18 +161,4 @@ private ComputationResult getCom .computeMillis(0) .build(); } - - class TestNodePropertyValues implements LongNodePropertyValues { - - @Override - public long size() { - return graphStore.nodeCount(); - } - - @Override - public long longValue(long nodeId) { - return nodeId; - } - } - } diff --git a/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java b/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java index 3252bcdfca9..437e1852b69 100644 --- a/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java @@ -59,7 +59,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return 42; } }; diff --git a/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java b/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java index 860ed636ded..0ee9279e587 100644 --- a/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java @@ -30,7 +30,7 @@ class ConsecutiveLongNodePropertyValuesTest { void shouldReturnConsecutiveIds() { LongNodePropertyValues nonConsecutiveIds = new LongNodePropertyValues() { @Override - public long size() { + public long nodeCount() { return 10; } @@ -40,12 +40,9 @@ public long longValue(long nodeId) { } }; - var consecutiveIds = new ConsecutiveLongNodePropertyValues( - nonConsecutiveIds, - 10 - ); + var consecutiveIds = new ConsecutiveLongNodePropertyValues(nonConsecutiveIds); - assertThat(consecutiveIds.size()).isEqualTo(10); + assertThat(consecutiveIds.nodeCount()).isEqualTo(nonConsecutiveIds.nodeCount()); for (int i = 0; i < 10; i++) { assertThat(consecutiveIds.longValue(i)).isEqualTo(i % 3); diff --git a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index 0761d9597d4..02379a65194 100644 --- a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -24,7 +24,6 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArray; import org.neo4j.gds.config.CommunitySizeConfig; -import org.neo4j.gds.config.ComponentSizeConfig; import org.neo4j.gds.config.ConcurrencyConfig; import org.neo4j.gds.config.ConsecutiveIdsConfig; import org.neo4j.gds.config.SeedConfig; @@ -44,39 +43,34 @@ private CommunityProcCompanion() {} public static NodePropertyValues nodeProperties( CONFIG config, String resultProperty, - LongNodePropertyValues nodeProperties, + LongNodePropertyValues propertyValues, Supplier seedPropertySupplier ) { - var consecutiveIds = config.consecutiveIds(); var isIncremental = config.isIncremental(); var resultPropertyEqualsSeedProperty = isIncremental && resultProperty.equals(config.seedProperty()); - LongNodePropertyValues result; - if (resultPropertyEqualsSeedProperty && !consecutiveIds) { - result = LongIfChangedNodePropertyValues.of(seedPropertySupplier.get(), nodeProperties); + var incrementalNodePropertyValues = LongIfChangedNodePropertyValues.of(seedPropertySupplier.get(), propertyValues); + return communitySizeFilter(incrementalNodePropertyValues, config); } else if (consecutiveIds && !isIncremental) { - result = new ConsecutiveLongNodePropertyValues( - nodeProperties, - nodeProperties.size() - ); + var sizeFilteredPropertyValues = communitySizeFilter(propertyValues, config); + return new ConsecutiveLongNodePropertyValues(sizeFilteredPropertyValues); } else { - result = nodeProperties; + return communitySizeFilter(propertyValues, config); } + } + private static LongNodePropertyValues communitySizeFilter( + LongNodePropertyValues result, + CONFIG config + ) { if (config instanceof CommunitySizeConfig) { var finalResult = result; result = ((CommunitySizeConfig) config) .minCommunitySize() .map(size -> applySizeFilter(finalResult, size, config.concurrency())) .orElse(result); - } else if (config instanceof ComponentSizeConfig) { - var finalResult = result; - result = ((ComponentSizeConfig) config) - .minComponentSize() - .map(size -> applySizeFilter(finalResult, size, config.concurrency())) - .orElse(result); } return result; @@ -88,7 +82,7 @@ private static LongNodePropertyValues applySizeFilter( int concurrency ) { var communitySizes = CommunityStatistics.communitySizes( - nodeProperties.size(), + nodeProperties.nodeCount(), nodeProperties::longValue, Pools.DEFAULT, concurrency @@ -111,8 +105,8 @@ private static class CommunitySizeFilter implements LongNodePropertyValues { } @Override - public long size() { - return properties.size(); + public long nodeCount() { + return properties.nodeCount(); } @Override diff --git a/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java b/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java index 629383e8e22..adca371d7a6 100644 --- a/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java @@ -22,6 +22,7 @@ import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.ProgressTimer; +import org.neo4j.gds.core.utils.paged.HugeIntArray; import org.neo4j.gds.core.write.NodePropertyExporter; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.AlgorithmSpecProgressTrackerProvider; @@ -71,7 +72,8 @@ public ComputationResultConsumer intermediateCommunity; - IntermediateCommunityNodeProperties(long size, LongFunction intermediateCommunity) { - this.size = size; + IntermediateCommunityNodeProperties(long nodeCount, long storedValues, LongFunction intermediateCommunity) { + this.nodeCount = nodeCount; + this.storedValues = storedValues; this.intermediateCommunity = intermediateCommunity; } @Override - public long size() { - return size; + public long nodeCount() { + return nodeCount; } @Override diff --git a/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenCompanion.java b/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenCompanion.java index c25a0bafc51..ab38def7ea1 100644 --- a/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenCompanion.java +++ b/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenCompanion.java @@ -38,6 +38,7 @@ static NodePropertyValues leidenNodeProperties if (config.includeIntermediateCommunities()) { return new IntermediateCommunityNodeProperties( + computationResult.graph().nodeCount(), leidenResult.communities().size(), leidenResult::getIntermediateCommunities ); diff --git a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java index cd06bd4aa90..cc4af49e6bb 100644 --- a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java +++ b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java @@ -54,7 +54,7 @@ static NodePropertyValues nodeProperties( return new LongArrayNodePropertyValues() { @Override - public long size() { + public long nodeCount() { return size; } diff --git a/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java b/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java index 538c704f510..5500182b4cb 100644 --- a/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java +++ b/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java @@ -76,7 +76,7 @@ public Value value(long nodeId) { } @Override - public long size() { - return newProperties.size(); + public long nodeCount() { + return Math.max(newProperties.nodeCount(), seedProperties.nodeCount()); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamProc.java b/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamProc.java index 5c56f6f1ffd..1d94071d536 100644 --- a/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamProc.java +++ b/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamProc.java @@ -92,7 +92,7 @@ protected NodePropertyValues nodeProperties(ComputationResult id < 5 ? id : 5 ); - var config = ConfigWithComponentSize.of(CypherMapWrapper.empty().withNumber("minComponentSize", 2L)); + var config = CommunityProcCompanionConfig.of(CypherMapWrapper.empty().withNumber("minCommunitySize", 2L)); var result = CommunityProcCompanion.nodeProperties( config, @@ -106,7 +110,7 @@ void shouldRestrictComponentSize() { () -> { throw new UnsupportedOperationException("Not implemented"); } ); - for (long i = 0L; i < result.size(); i++) { + for (long i = 0L; i < result.nodeCount(); i++) { if (i < 5) { assertThat(result.longValue(i)).isEqualTo(inputProperties.longValue(i)); @@ -118,6 +122,92 @@ void shouldRestrictComponentSize() { } } + @Test + void shouldWorkWithMinComponentAndConsecutive() { + long[] values = new long[]{20, 20, 200, 10, 10, 90, 50, 10, 50, 50, 50}; + Long[] returnedValues = new Long[]{null, null, null, 0L, 0L, null, 1L, 0L, 1L, 1L, 1L}; + + LongNodePropertyValues inputProperties = new TestNodePropertyValues(11, id -> values[(int) id]); + + var config = CommunityProcCompanionConfig.of(CypherMapWrapper + .empty() + .withNumber("minCommunitySize", 3L) + .withBoolean("consecutiveIds", true)); + + var result = CommunityProcCompanion.nodeProperties( + config, + "seed", + inputProperties, + () -> {throw new UnsupportedOperationException("Not implemented");} + ); + + + for (long i = 0L; i < result.nodeCount(); i++) { + int ii = (int) i; + if (returnedValues[ii] != null) { + assertThat(result.value(i).asObject()).isEqualTo(returnedValues[ii]); + assertThat(result.longValue(i)).isEqualTo(returnedValues[ii]); + } else { + assertThat(result.value(i)).isNull(); + assertThat(result.longValue(i)).isLessThan(0L); + } + + } + + } + + @Test + void minComponentSizeWithSparseProperties() { + var inputBuilder = HugeSparseLongArray.builder(Long.MIN_VALUE); + inputBuilder.set(1, 42); + inputBuilder.set(2, 99); + inputBuilder.set(3, 42); + var input = inputBuilder.build(); + + // we mimic the sparseness here through size > valueStored + LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, input::get); + + var config = CommunityProcCompanionConfig.of(CypherMapWrapper.empty().withNumber("minCommunitySize", 2L)); + + var filteredProperties = CommunityProcCompanion.nodeProperties( + config, + "seed", + sparseProperties, + () -> { throw new UnsupportedOperationException("Not implemented"); } + ); + + assertThat(filteredProperties.value(0)).isNull(); + assertThat(filteredProperties.value(1)).isEqualTo(Values.longValue(42)); + assertThat(filteredProperties.value(2)).isNull(); + assertThat(filteredProperties.value(3)).isEqualTo(Values.longValue(42)); + } + + @Test + void consecutiveIdsWithSparseProperties() { + var inputBuilder = HugeSparseLongArray.builder(Long.MIN_VALUE); + inputBuilder.set(1, 42); + inputBuilder.set(2, 99); + inputBuilder.set(3, 42); + var input = inputBuilder.build(); + + // we mimic the sparseness here through size > valueStored + LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, input::get); + + var config = CommunityProcCompanionConfig.of(CypherMapWrapper.create(Map.of("consecutiveIds", true))); + + var filteredProperties = CommunityProcCompanion.nodeProperties( + config, + "seed", + sparseProperties, + () -> { throw new UnsupportedOperationException("Not implemented"); } + ); + + assertThat(filteredProperties.value(0)).isEqualTo(Values.longValue(0)); + assertThat(filteredProperties.value(1)).isEqualTo(Values.longValue(1)); + assertThat(filteredProperties.value(2)).isEqualTo(Values.longValue(2)); + assertThat(filteredProperties.value(3)).isEqualTo(Values.longValue(1)); + } + private static final class TestNodePropertyValues implements LongNodePropertyValues { private final long size; private final LongToLongFunction transformer; @@ -131,7 +221,7 @@ private TestNodePropertyValues( } @Override - public long size() { + public long nodeCount() { return size; } @@ -141,17 +231,33 @@ public long longValue(long nodeId) { } } - @Configuration - interface CommunityProcCompanionConfig extends ConsecutiveIdsConfig, SeedConfig, ConcurrencyConfig { - static CommunityProcCompanionConfig of(CypherMapWrapper map) { - return new CommunityProcCompanionConfigImpl(map); + private static final class TestSparseNodePropertyValues implements LongNodePropertyValues { + private final long size; + private final LongToLongFunction transformer; + + private TestSparseNodePropertyValues( + long size, + LongToLongFunction transformer + ) { + this.size = size; + this.transformer = transformer; + } + + @Override + public long nodeCount() { + return size; + } + + @Override + public long longValue(long nodeId) { + return transformer.applyAsLong(nodeId); } } @Configuration - interface ConfigWithComponentSize extends ConsecutiveIdsConfig, SeedConfig, ComponentSizeConfig, ConcurrencyConfig { - static ConfigWithComponentSize of(CypherMapWrapper map) { - return new ConfigWithComponentSizeImpl(map); + interface CommunityProcCompanionConfig extends ConsecutiveIdsConfig, SeedConfig, ConcurrencyConfig, CommunitySizeConfig { + static CommunityProcCompanionConfig of(CypherMapWrapper map) { + return new CommunityProcCompanionConfigImpl(map); } } } diff --git a/proc/community/src/test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java b/proc/community/src/test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java index 90bd4e0a1de..776bc64607c 100644 --- a/proc/community/src/test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java @@ -110,7 +110,7 @@ void yields() { assertThat(resultRow.get("averageDistanceToCentroid")) .isNotNull() - .asInstanceOf(DOUBLE); + .asInstanceOf(DOUBLE).isNotEqualTo(0.0d); var centroids = resultRow.get("centroids"); assertThat(centroids) @@ -126,7 +126,7 @@ void yields() { } assertThat(resultRow.get("averageSilhouette")) .isNotNull() - .asInstanceOf(DOUBLE).isGreaterThanOrEqualTo(-1d).isLessThanOrEqualTo(1d); + .asInstanceOf(DOUBLE).isGreaterThanOrEqualTo(-1d).isLessThanOrEqualTo(1d).isNotEqualTo(0.0d); } return true; diff --git a/proc/community/src/test/java/org/neo4j/gds/louvain/LouvainWriteProcTest.java b/proc/community/src/test/java/org/neo4j/gds/louvain/LouvainWriteProcTest.java index f9dae068056..572fc560dea 100644 --- a/proc/community/src/test/java/org/neo4j/gds/louvain/LouvainWriteProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/louvain/LouvainWriteProcTest.java @@ -209,12 +209,12 @@ void zeroCommunitiesInEmptyGraph() { static Stream communitySizeInputs() { return Stream.of( // configuration | expectedCommunityCount | expectedCommunityIds - Arguments.of(Map.of("minCommunitySize", 1), 3L, new Long[] {11L, 13L, 14L}), - Arguments.of(Map.of("minCommunitySize", 3), 3L, new Long[] {14L, 13L}), - Arguments.of(Map.of("minCommunitySize", 1, "consecutiveIds", true), 3L, new Long[] {0L, 1L, 2L}), - Arguments.of(Map.of("minCommunitySize", 3, "consecutiveIds", true), 3L, new Long[] {1L, 2L}), - Arguments.of(Map.of("minCommunitySize", 1, "seedProperty", SEED_PROPERTY), 3L, new Long[] {1L, 2L, 42L}), - Arguments.of(Map.of("minCommunitySize", 3, "seedProperty", SEED_PROPERTY), 3L, new Long[] {2L, 42L}) + Arguments.of(Map.of("minCommunitySize", 1), 3L, new Long[]{11L, 13L, 14L}), + Arguments.of(Map.of("minCommunitySize", 3), 3L, new Long[]{14L, 13L}), + Arguments.of(Map.of("minCommunitySize", 1, "consecutiveIds", true), 3L, new Long[]{0L, 1L, 2L}), + Arguments.of(Map.of("minCommunitySize", 3, "consecutiveIds", true), 3L, new Long[]{0L, 1L}), + Arguments.of(Map.of("minCommunitySize", 1, "seedProperty", SEED_PROPERTY), 3L, new Long[]{1L, 2L, 42L}), + Arguments.of(Map.of("minCommunitySize", 3, "seedProperty", SEED_PROPERTY), 3L, new Long[]{2L, 42L}) ); } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java index b5de01300bb..066ccec4f13 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java @@ -40,7 +40,7 @@ public float[] floatArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return nodeCount; } }; diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java index 99026545670..99e63730308 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java @@ -46,7 +46,7 @@ public static DoubleArrayNodePropertyValues getN return new DoubleArrayNodePropertyValues() { @Override - public long size() { + public long nodeCount() { return size; } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java index e8ed5edbb79..42c21637171 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java @@ -40,7 +40,7 @@ public double[] doubleArrayValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return nodeCount; } }; diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageBaseProcTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageBaseProcTest.java index 125525117ca..f4d91beef45 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageBaseProcTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageBaseProcTest.java @@ -29,7 +29,6 @@ import org.neo4j.gds.NodeProjections; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.catalog.GraphProjectProc; @@ -165,18 +164,14 @@ static Stream missingNodeProperties() { Arguments.of( ImmutableGraphProjectFromStoreConfig.builder() .graphName("implicitWeightedGraph") - .nodeProjections(NodeProjections - .builder() - .putProjection( - NodeLabel.of("King"), - NodeProjection.of( - "King", - PropertyMappings.of( - PropertyMapping.of("age") - ) - ) - ) - .build()) + .nodeProjections(NodeProjections.single( + NodeLabel.of("King"), + NodeProjection.builder() + .label("King") + .addProperty( + PropertyMapping.of("age") + ).build() + )) .relationshipProjections(RelationshipProjections.fromString("REL") ).build(), List.of("birth_year", "death_year"), @@ -186,19 +181,15 @@ static Stream missingNodeProperties() { Arguments.of( ImmutableGraphProjectFromStoreConfig.builder() .graphName("implicitWeightedGraph") - .nodeProjections(NodeProjections - .builder() - .putProjection( - NodeLabel.of("King"), - NodeProjection.of( - "King", - PropertyMappings.of( - PropertyMapping.of("age"), - PropertyMapping.of("birth_year") - ) - ) - ) - .build()) + .nodeProjections(NodeProjections.single( + NodeLabel.of("King"), + NodeProjection.builder() + .label("King") + .addProperties( + PropertyMapping.of("age"), + PropertyMapping.of("birth_year") + ).build() + )) .relationshipProjections(RelationshipProjections.fromString("REL") ).build(), List.of("death_year"), diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageIntegrationTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageIntegrationTest.java index 05b432bd1e9..59d8108d528 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageIntegrationTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageIntegrationTest.java @@ -113,7 +113,7 @@ private void dropModel() { ), "creationTime", isA(ZonedDateTime.class), "trainConfig", allOf( - aMapWithSize(19), + aMapWithSize(20), hasEntry("modelName", modelName), hasEntry("aggregator", "MEAN"), hasEntry("activationFunction", "SIGMOID") diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java index 2ba78761f65..f4251641d00 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java @@ -25,7 +25,6 @@ import org.neo4j.gds.GdsCypher; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.TestProcedureRunner; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.schema.GraphSchema; @@ -101,16 +100,18 @@ void runsTrainingOnMultiLabelGraph() { .graphProject() .withNodeLabel( "A", - NodeProjection.of("A", PropertyMappings.of( - PropertyMapping.of("a1"), - PropertyMapping.of("a2") - )) + NodeProjection + .builder() + .label("A") + .addProperties(PropertyMapping.of("a1"), PropertyMapping.of("a2")) + .build() ).withNodeLabel( "B", - NodeProjection.of("B", PropertyMappings.of( - PropertyMapping.of("b1"), - PropertyMapping.of("b2") - )) + NodeProjection + .builder() + .label("B") + .addProperties(PropertyMapping.of("b1"), PropertyMapping.of("b2")) + .build() ) .withAnyRelationshipType() .yields(); diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java index 58a10d577e9..439164dec94 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java @@ -108,10 +108,14 @@ protected void updateGraphStore( Collection labelFilter = computationResult.algorithm().labelFilter().predictNodeLabels(); var graph = graphStore.getGraph(labelFilter); - var concurrency = computationResult.config().concurrency(); + var config = computationResult.config(); + var concurrency = config.concurrency(); + var mutateRelationshipType = RelationshipType.of(config.mutateRelationshipType()); + var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .aggregation(Aggregation.SINGLE) .nodes(graph) + .relationshipType(mutateRelationshipType) .orientation(Orientation.UNDIRECTED) .addPropertyConfig(GraphFactory.PropertyConfig.of(computationResult.config().mutateProperty())) .concurrency(concurrency) @@ -134,10 +138,11 @@ protected void updateGraphStore( var relationships = relationshipsBuilder.build(); - var config = computationResult.config(); + + computationResult .graphStore() - .addRelationshipType(RelationshipType.of(config.mutateRelationshipType()), relationships); + .addRelationshipType(relationships); resultBuilder.withRelationshipsWritten(relationships.topology().elementCount()); } }; diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java index b0d05fc5451..e741daf872d 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java @@ -97,7 +97,7 @@ protected List nodePropertyList(ComputationResult { var properties = new DoubleArrayNodePropertyValues() { @Override - public long size() { + public long nodeCount() { return computationResult.graph().nodeCount(); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java index 283f6e9bc2c..c7c4c5e0af4 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java @@ -98,7 +98,7 @@ protected List nodePropertyList(ComputationResult { var properties = new DoubleArrayNodePropertyValues() { @Override - public long size() { + public long nodeCount() { return computationResult.graph().nodeCount(); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java index ac79e6e6aeb..9860863fead 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java @@ -66,13 +66,13 @@ public Stream mutate( @Override protected NodePropertyValues nodeProperties(ComputationResult computationResult) { - var size = computationResult.graph().nodeCount(); + var nodeCount = computationResult.graph().nodeCount(); var predictedPropertyValues = computationResult.result(); return new DoubleNodePropertyValues() { @Override - public long size() { - return size; + public long nodeCount() { + return nodeCount; } @Override diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationshipsMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationshipsMutateProc.java index 134faff9f9e..eb7299f99ac 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationshipsMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationshipsMutateProc.java @@ -85,8 +85,8 @@ protected void updateGraphStore( var remainingRels = splitResult.remainingRels().build(); var config = computationResult.config(); - graphStore.addRelationshipType(config.remainingRelationshipType(), remainingRels); - graphStore.addRelationshipType(config.holdoutRelationshipType(), selectedRels); + graphStore.addRelationshipType(remainingRels); + graphStore.addRelationshipType(selectedRels); long holdoutWritten = selectedRels.topology().elementCount(); long remainingWritten = remainingRels.topology().elementCount(); diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainProcTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainProcTest.java index 6ba65be0705..acc1538ddca 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainProcTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainProcTest.java @@ -184,7 +184,7 @@ void trainAModel() { Matchers.hasKey("bestParameters") ), "trainMillis", greaterThan(-1L), - "configuration", aMapWithSize(12) + "configuration", aMapWithSize(13) )) ); @@ -257,7 +257,7 @@ void trainOnNodeLabelFilteredGraph() { Matchers.hasKey("bestParameters") ), "trainMillis", greaterThan(-1L), - "configuration", aMapWithSize(12) + "configuration", aMapWithSize(13) )) ); GraphStore graphStore = GraphStoreCatalog.get(getUsername(), DatabaseId.of(db), GRAPH_NAME).graphStore(); diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProcTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProcTest.java index 6ea8ce5b85c..8b6136f4a6b 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProcTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProcTest.java @@ -105,7 +105,7 @@ void mutate() { Graph mutatedGraph = GraphStoreCatalog.get(TEST_USERNAME, DatabaseId.of(db), GRAPH_NAME).graphStore().getUnion(); assertThat(mutatedGraph.availableNodeProperties()).isEqualTo(Set.of("a", "b", "class")); - assertThat(mutatedGraph.nodeProperties("class").size()).isEqualTo(5); + assertThat(mutatedGraph.nodeProperties("class").nodeCount()).isEqualTo(5); } @Test @@ -132,8 +132,8 @@ void mutateWithProbabilities(){ Graph mutatedGraph = GraphStoreCatalog.get(TEST_USERNAME, DatabaseId.of(db), GRAPH_NAME).graphStore().getUnion(); assertThat(mutatedGraph.availableNodeProperties()).isEqualTo(Set.of("a", "b", "class", "probabilities")); - assertThat(mutatedGraph.nodeProperties("class").size()).isEqualTo(5); - assertThat(mutatedGraph.nodeProperties("probabilities").size()).isEqualTo(5); + assertThat(mutatedGraph.nodeProperties("class").nodeCount()).isEqualTo(5); + assertThat(mutatedGraph.nodeProperties("probabilities").nodeCount()).isEqualTo(5); assertThat(mutatedGraph.nodeProperties("probabilities").doubleArrayValue(0)) .containsExactly(new double[]{0.012080865612605783, 0.9879191343873942}, Offset.offset(1e-6)); } diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainProcTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainProcTest.java index 5b131e0fa4a..555f60ac994 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainProcTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainProcTest.java @@ -268,7 +268,7 @@ void train() { "configuration", Matchers.allOf( Matchers.hasEntry("pipeline", PIPELINE_NAME), Matchers.hasEntry("modelName", MODEL_NAME), - aMapWithSize(11) + aMapWithSize(12) ), "modelSelectionStats", modelSelectionStatsCheck ) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainProcTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainProcTest.java index ecf128ec9d2..5f4ddc623df 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainProcTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainProcTest.java @@ -233,7 +233,7 @@ void train() { "configuration", Matchers.allOf( Matchers.hasEntry("pipeline", PIPELINE_NAME), Matchers.hasEntry("modelName", MODEL_NAME), - aMapWithSize(11) + aMapWithSize(12) ), "modelSelectionStats",modelSelectionStatsCheck ) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProcTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProcTest.java index 330ef31f393..ebe31ffb283 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProcTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProcTest.java @@ -109,7 +109,7 @@ void mutate() { assertThat(graphStore.nodePropertyKeys()).contains("p"); var propertyValues = graphStore.nodeProperty("p").values(); - assertThat(propertyValues.size()).isEqualTo(5); + assertThat(propertyValues.nodeCount()).isEqualTo(5); assertThat(propertyValues.doubleValue(idFunction.of("n0"))).isEqualTo(99.3); assertThat(propertyValues.doubleValue(idFunction.of("n1"))).isEqualTo(100.0); diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java index 73cfba0b00c..3b7bcfd7fee 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java @@ -56,6 +56,7 @@ protected void updateGraphStore( var relationshipsBuilder = GraphFactory .initRelationshipsBuilder() + .relationshipType(mutateRelationshipType) .nodes(computationResult.graph()) .addPropertyConfig(GraphFactory.PropertyConfig.of(TOTAL_COST_KEY)) .orientation(Orientation.NATURAL) @@ -75,6 +76,6 @@ protected void updateGraphStore( computationResult .graphStore() - .addRelationshipType(mutateRelationshipType, relationships); + .addRelationshipType(relationships); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java index bb5a9635091..cf571b1fd4a 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java @@ -45,7 +45,8 @@ public Stream consume( var shouldReturnPath = executionContext.callContext() .outputFields() - .anyMatch(field -> toLowerCaseWithLocale(field).equals("path")); + .anyMatch(field -> toLowerCaseWithLocale(field).equals("path")) + && computationResult.graphStore().capabilities().canWriteToDatabase(); var resultBuilder = new StreamResult.Builder(graph, executionContext.transaction().internalTransaction()); diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java index 77b4f5f43b6..86d69110b5f 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java @@ -72,15 +72,15 @@ public ComputationResultConsumer. + */ +package org.neo4j.gds.paths.sourcetarget; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.beta.generator.GraphGenerateProc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; + + class ShortestPathRandomGraphProcTest extends BaseProcTest { + + @BeforeEach + void setup() throws Exception { + registerProcedures( + ShortestPathYensStreamProc.class, + ShortestPathDijkstraStreamProc.class, + ShortestPathAStarStreamProc.class, + GraphGenerateProc.class + ); + } + + @ParameterizedTest + @ValueSource( + strings = { + "CALL gds.shortestPath.yens.stream('graph',{k:1, sourceNode:0, targetNode:1})", + "CALL gds.shortestPath.dijkstra.stream('graph',{sourceNode:0, targetNode:1})" + } + ) + void shouldWorkWithRandomGraph(String query) { + + runQuery("CALL gds.beta.graph.generate('graph',10,10)"); + assertThatNoException().isThrownBy(() -> { + + long rowCount = runQueryWithRowConsumer(query, result -> { + assertThat(result.get("path")).isNull(); + }); + assertThat(rowCount).isEqualTo(1L); + + }); + + + } + +} diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java new file mode 100644 index 00000000000..a176c6aa5de --- /dev/null +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.paths.sourcetarget; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.IdFunction; +import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.extension.Neo4jGraph; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class YensTestWithDifferentProjections extends BaseProcTest { + + @Neo4jGraph + private static final String DB_CYPHER = + "CREATE (a:CITY {cityid:0}), " + + "(b:CITY {cityid:1}), " + + "(c:CITY {cityid:2}), " + + "(d:CITY {cityid:3}), " + + "(e:CITY {cityid:4}), " + + "(f:CITY {cityid:5}), " + + "(a)-[:ROAD]->(b), " + + "(a)-[:ROAD]->(b), " + + "(b)-[:ROAD]->(c), " + + "(b)-[:ROAD]->(d), " + + "(c)-[:ROAD]->(f), " + + "(d)-[:ROAD]->(e), " + + "(e)-[:ROAD]->(c), " + + "(e)-[:ROAD]->(f), " + + "(a)-[:PATH]->(b), " + + "(d)-[:PATH]->(e), " + + "(d)-[:PATH]->(e)"; + + @Inject + IdFunction idFunction; + + @BeforeEach + void setup() throws Exception { + registerProcedures( + ShortestPathYensStreamProc.class, + GraphProjectProc.class + ); + } + + + @ParameterizedTest + @ValueSource(strings = { + "CALL gds.graph.project('g', '*', {TYPE: {type: '*', aggregation: 'SINGLE'}})", + "CALL gds.graph.project.cypher('g', 'MATCH (n) RETURN id(n) AS id', 'MATCH (n)-[r]->(m) RETURN DISTINCT id(n) AS source, id(m) AS target')" + }) + void shouldWorkWithDifferentProjections(String projectionQuery) { + + runQuery(projectionQuery); + String yensQuery = "MATCH (source), (target) " + + "WHERE source.cityid=0 AND target.cityid=5 " + + "CALL gds.shortestPath.yens.stream(" + + " 'g', " + + " {sourceNode:source, targetNode:target, k:3} " + + ") " + + "YIELD nodeIds RETURN nodeIds "; + + Collection encounteredPaths = new HashSet<>(); + runQuery(yensQuery, result -> { + assertThat(result.columns()).containsExactlyInAnyOrder("nodeIds"); + + while (result.hasNext()) { + var next = result.next(); + var currentPath = (List) next.get("nodeIds"); + long[] pathToArray = currentPath.stream().mapToLong(l -> l).toArray(); + encounteredPaths.add(pathToArray); + } + + return true; + }); + + long[] nodes = new long[]{idFunction.of("a"), idFunction.of("b"), idFunction.of("c"), idFunction.of("d"), idFunction.of( + "e"), idFunction.of("f")}; + assertThat(encounteredPaths).containsExactlyInAnyOrder( + new long[]{nodes[0], nodes[1], nodes[3], nodes[4], nodes[2], nodes[5]}, + new long[]{nodes[0], nodes[1], nodes[3], nodes[4], nodes[5]}, + new long[]{nodes[0], nodes[1], nodes[2], nodes[5]} + ); + } + +} + diff --git a/proc/pregel/build.gradle b/proc/pregel/build.gradle index a3b821a38d6..babfad1e869 100644 --- a/proc/pregel/build.gradle +++ b/proc/pregel/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation project(':executor') implementation project(':proc-common') implementation project(':string-formatting') - + implementation project(':graph-schema-api') api project(':pregel') testAnnotationProcessor project(':annotations') diff --git a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java index fdf2582a45e..74add664fcc 100644 --- a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java +++ b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java @@ -20,34 +20,115 @@ package org.neo4j.gds.pregel.proc; import org.neo4j.gds.Algorithm; +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.nodes.DoubleArrayNodePropertyValues; import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.beta.indexInverse.InverseRelationshipsAlgorithmFactory; +import org.neo4j.gds.beta.indexInverse.InverseRelationshipsConfigImpl; import org.neo4j.gds.beta.pregel.PregelConfig; import org.neo4j.gds.beta.pregel.PregelResult; import org.neo4j.gds.beta.pregel.PregelSchema; import org.neo4j.gds.core.utils.paged.HugeObjectArray; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; import org.neo4j.gds.core.write.ImmutableNodeProperty; import org.neo4j.gds.core.write.NodeProperty; import org.neo4j.gds.executor.ComputationResult; +import org.neo4j.gds.executor.validation.AfterLoadValidation; +import org.neo4j.gds.executor.validation.ValidationConfiguration; +import org.neo4j.gds.utils.StringJoining; +import org.neo4j.logging.Log; +import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -final class PregelBaseProc { +public final class PregelBaseProc { - static , CONFIG extends PregelConfig> - List nodeProperties( - ComputationResult computationResult, - String propertyPrefix + public static ValidationConfiguration ensureIndexValidation( + Log log, TaskRegistryFactory taskRegistryFactory + ) { + return new ValidationConfiguration<>() { + @Override + public List> afterLoadValidations() { + return List.of( + (graphStore, graphProjectConfig, config) -> ensureDirectedRelationships( + graphStore, config.internalRelationshipTypes(graphStore) + ), + (graphStore, graphProjectConfig, config) -> ensureInverseIndexesExist(graphStore, + config.internalRelationshipTypes(graphStore), + config.concurrency(), + log, + taskRegistryFactory + ) + ); + } + }; + } + + static void ensureInverseIndexesExist( + GraphStore graphStore, + Collection relationshipTypes, + int concurrency, + Log log, + TaskRegistryFactory taskRegistryFactory + ) { + var relationshipTypesWithoutIndex = relationshipTypes + .stream() + .filter(relType -> !graphStore.inverseIndexedRelationshipTypes().contains(relType)) + .map(RelationshipType::name) + .collect(Collectors.toList()); + + if (relationshipTypesWithoutIndex.isEmpty()) { + return; + } + + var inverseConfig = InverseRelationshipsConfigImpl + .builder() + .concurrency(concurrency) + .relationshipTypes(relationshipTypesWithoutIndex) + .build(); + + new InverseRelationshipsAlgorithmFactory() + .build(graphStore, inverseConfig, log, taskRegistryFactory) + .compute() + .forEach((relationshipType, inverseIndex) -> graphStore.addInverseIndex( + relationshipType, + inverseIndex.topology(), + inverseIndex.properties() + )); + } + + static void ensureDirectedRelationships(GraphStore graphStore, Collection relationshipTypes) { + var relationshipSchema = graphStore.schema().relationshipSchema(); + var undirectedTypes = relationshipTypes + .stream() + .filter(relationshipSchema::isUndirected) + .map(RelationshipType::name) + .collect(Collectors.toList()); + + if (!undirectedTypes.isEmpty()) { + throw new IllegalArgumentException(String.format( + Locale.US, + "This algorithm requires a directed graph, but the following configured relationship types are undirected: %s.", + StringJoining.join(undirectedTypes) + )); + } + } + + static , CONFIG extends PregelConfig> List nodeProperties( + ComputationResult computationResult, String propertyPrefix ) { var compositeNodeValue = computationResult.result().nodeValues(); var schema = compositeNodeValue.schema(); // TODO change this to generic prefix setting - return schema.elements() + return schema + .elements() .stream() .filter(element -> element.visibility() == PregelSchema.Visibility.PUBLIC) .map(element -> { @@ -62,23 +143,16 @@ List nodeProperties( nodePropertyValues = compositeNodeValue.doubleProperties(propertyKey).asNodeProperties(); break; case LONG_ARRAY: - nodePropertyValues = new HugeObjectArrayLongArrayPropertyValues( - compositeNodeValue.longArrayProperties(propertyKey) - ); + nodePropertyValues = new HugeObjectArrayLongArrayPropertyValues(compositeNodeValue.longArrayProperties(propertyKey)); break; case DOUBLE_ARRAY: - nodePropertyValues = new HugeObjectArrayDoubleArrayPropertyValues( - compositeNodeValue.doubleArrayProperties(propertyKey) - ); + nodePropertyValues = new HugeObjectArrayDoubleArrayPropertyValues(compositeNodeValue.doubleArrayProperties(propertyKey)); break; default: throw new IllegalArgumentException("Unsupported property type: " + element.propertyType()); } - return ImmutableNodeProperty.of( - formatWithLocale("%s%s", propertyPrefix, propertyKey), - nodePropertyValues - ); + return ImmutableNodeProperty.of(formatWithLocale("%s%s", propertyPrefix, propertyKey), nodePropertyValues); }).collect(Collectors.toList()); } @@ -90,7 +164,8 @@ static class HugeObjectArrayLongArrayPropertyValues implements LongArrayNodeProp HugeObjectArrayLongArrayPropertyValues(HugeObjectArray longArrays) {this.longArrays = longArrays;} @Override - public long size() { + public long nodeCount() { + // Its backed by a dense array return longArrays.size(); } @@ -106,7 +181,8 @@ static class HugeObjectArrayDoubleArrayPropertyValues implements DoubleArrayNode HugeObjectArrayDoubleArrayPropertyValues(HugeObjectArray doubleArrays) {this.doubleArrays = doubleArrays;} @Override - public long size() { + public long nodeCount() { + // its backed by dense array return doubleArrays.size(); } diff --git a/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java b/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java new file mode 100644 index 00000000000..3fec11bbaf3 --- /dev/null +++ b/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pregel.proc; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; +import org.neo4j.logging.NullLog; + +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@GdlExtension +class PregelBaseProcTest { + + @GdlGraph + @GdlGraph(graphNamePrefix = "undirected", orientation = Orientation.UNDIRECTED) + public static String GDL = + "CREATE " + + " ()-[:REL]->()," + + " ()-[:REL2]->(),"; + + @Inject + GraphStore graphStore; + + @Inject + GraphStore undirectedGraphStore; + + static Stream relTypeCombinations() { + var rel = RelationshipType.of("REL"); + var rel2 = RelationshipType.of("REL2"); + return Stream.of(Arguments.arguments(List.of(rel)), + Arguments.arguments(List.of(rel2)), + Arguments.arguments(List.of(rel, rel2)) + ); + } + + @ParameterizedTest + @MethodSource("relTypeCombinations") + void shouldGenerateInverseIndexes(List relTypes) { + PregelBaseProc.ensureInverseIndexesExist(graphStore, + relTypes, + 4, + NullLog.getInstance(), + TaskRegistryFactory.empty() + ); + assertThat(graphStore.inverseIndexedRelationshipTypes()).containsExactlyElementsOf(relTypes); + } + + @Test + void shouldThrowOnUndirectedRelTypes() { + assertThatThrownBy(() -> PregelBaseProc.ensureDirectedRelationships( + undirectedGraphStore, + RelationshipType.listOf("REL", "REL2") + )).hasMessage( + "This algorithm requires a directed graph, but the following configured relationship types are undirected: ['REL', 'REL2']." + ); + } +} diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateProc.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateProc.java index 27e93259185..883f61ebf54 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateProc.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateProc.java @@ -121,11 +121,12 @@ public ComputationResultConsumer computationResult, SimilarityGraphResult similarityGraphResult, String relationshipPropertyKey, @@ -129,11 +132,13 @@ private SingleTypeRelationships getRelationships( ) { SingleTypeRelationships resultRelationships; + if (similarityGraphResult.isTopKGraph()) { TopKGraph topKGraph = (TopKGraph) similarityGraphResult.similarityGraph(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(topKGraph) + .relationshipType(relationshipType) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of(relationshipPropertyKey)) .concurrency(1) @@ -167,6 +172,7 @@ private SingleTypeRelationships getRelationships( var similarityGraph = (HugeGraph) similarityGraphResult.similarityGraph(); resultRelationships = SingleTypeRelationships.of( + relationshipType, similarityGraph.relationshipTopology(), similarityGraph.schema().direction(), similarityGraph.relationshipProperties(), diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateProc.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateProc.java index 2f0b7d90f5c..b7dd835af6c 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateProc.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateProc.java @@ -139,8 +139,8 @@ public ComputationResultConsumer computationResult, + RelationshipType relationshipType, ComputationResult computationResult, SimilarityGraphResult similarityGraphResult, String relationshipPropertyKey, SimilarityProc.SimilarityResultBuilder resultBuilder @@ -155,6 +155,7 @@ private SingleTypeRelationships getRelationships( RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(topKGraph) + .relationshipType(relationshipType) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of(relationshipPropertyKey)) .concurrency(1) @@ -188,6 +189,7 @@ private SingleTypeRelationships getRelationships( HugeGraph similarityGraph = (HugeGraph) similarityGraphResult.similarityGraph(); relationships = SingleTypeRelationships.of( + relationshipType, similarityGraph.relationshipTopology(), similarityGraph.schema().direction(), similarityGraph.relationshipProperties(), diff --git a/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java b/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java index 968574f76a4..d21b7d202d0 100644 --- a/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java +++ b/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java @@ -30,6 +30,8 @@ import java.util.List; import java.util.Map; +import static org.assertj.core.api.Assertions.assertThat; + class KnnStreamProcTest extends KnnProcTest { private static final Collection EXPECTED = new HashSet<>(); @@ -94,4 +96,28 @@ void shouldStreamWithFilteredNodes() { Map.of("node1", 7L, "node2", 6L, "similarity", 1.0) )); } + + @Test + void computeOverSparseNodeProperties() { + String nodeCreateQuery = + "CREATE " + + " (alice:Person {grades: [24, 4]})" + + " ,(eve:Person)" + + " ,(bob:Foo {grades: [24, 4, 42]})"; + + runQuery(nodeCreateQuery); + + String createQuery = "CALL gds.graph.project('graph', " + + "'Person', '*', {nodeProperties: {grades: {defaultValue: [1, 1]}}})"; + runQuery(createQuery); + + String algoQuery = GdsCypher.call("graph") + .algo("gds.knn") + .streamMode() + .addParameter("nodeLabels", List.of("Person")) + .addParameter("nodeProperties", List.of("grades")) + .yields("node1", "node2", "similarity"); + + runQueryWithResultConsumer(algoQuery, result -> assertThat(result.stream().count()).isEqualTo(2)); + } } diff --git a/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java b/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java index a8c0c6f1e4d..c5316055b87 100644 --- a/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java +++ b/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java @@ -309,6 +309,7 @@ private static void configInfo(Config config, Consumer builder) { trySetting("dbms.tx_state.max_off_heap_memory", config, builder); trySetting("dbms.memory.off_heap.max_size", config, builder); trySetting("server.memory.off_heap.max_size", config, builder); + trySetting("server.memory.off_heap.transaction_max_size", config, builder); trySetting("dbms.memory.transaction.global_max_size", config, builder); trySetting("dbms.memory.transaction.total.max", config, builder); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java index 93521804ba4..5776b18d176 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java @@ -50,7 +50,7 @@ void shouldReturnGradleVersion() throws IOException { fail("Could not find version in file: " + file.toAbsolutePath())); var buildInfo = BuildInfoProperties.get(); - assertEquals(version, buildInfo.gdsVersion()); + assertThat(buildInfo.gdsVersion()).startsWith(version); } @Test @@ -106,7 +106,7 @@ void loadFromPropertiesRequiresVersion() { } private Optional findVersion(Path file) throws IOException { - Pattern pattern = Pattern.compile(".*gdsVersion = '(\\d\\.\\d\\.\\d+(-alpha\\d+|-beta\\d+)?)'.*"); + Pattern pattern = Pattern.compile(".*gdsBaseVersion = '(\\d\\.\\d\\.\\d+(-alpha\\d+|-beta\\d+)?)'.*"); return Files.lines(file, StandardCharsets.UTF_8) .flatMap(line -> { var matcher = pattern.matcher(line); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index e6c11a96723..a49229caf17 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -66,10 +66,35 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.3", "Neo4j Settings 5.3 (placeholder)", - "Neo4j DEV", - "Neo4j DEV (placeholder)", - "Neo4j Settings DEV", - "Neo4j Settings DEV (placeholder)", + "Neo4j 5.4", + "Neo4j 5.4 (placeholder)", + "Neo4j Settings 5.4", + "Neo4j Settings 5.4 (placeholder)", + + "Neo4j 5.5", + "Neo4j 5.5 (placeholder)", + "Neo4j Settings 5.5", + "Neo4j Settings 5.5 (placeholder)", + + "Neo4j 5.6", + "Neo4j 5.6 (placeholder)", + "Neo4j Settings 5.6", + "Neo4j Settings 5.6 (placeholder)", + + "Neo4j 5.7", + "Neo4j 5.7 (placeholder)", + "Neo4j Settings 5.7", + "Neo4j Settings 5.7 (placeholder)", + + "Neo4j 5.8", + "Neo4j 5.8 (placeholder)", + "Neo4j Settings 5.8", + "Neo4j Settings 5.8 (placeholder)", + + "Neo4j 5.9", + "Neo4j 5.9 (placeholder)", + "Neo4j Settings 5.9", + "Neo4j Settings 5.9 (placeholder)", "Neo4j RC", "Neo4j RC (placeholder)", @@ -150,14 +175,60 @@ void testSysInfoProc() throws IOException { "Neo4j 5.3" ); break; - case V_Dev: + case V_5_4: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.4 (placeholder)", + "Neo4j Settings 5.4", + "Neo4j 5.4 (placeholder)", + "Neo4j 5.4" + ); + break; + case V_5_5: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.5 (placeholder)", + "Neo4j Settings 5.5", + "Neo4j 5.5 (placeholder)", + "Neo4j 5.5" + ); + break; + case V_5_6: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.6 (placeholder)", + "Neo4j Settings 5.6", + "Neo4j 5.6 (placeholder)", + "Neo4j 5.6" + ); + break; + case V_5_7: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.7 (placeholder)", + "Neo4j Settings 5.7", + "Neo4j 5.7 (placeholder)", + "Neo4j 5.7" + ); + break; + case V_5_8: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.8 (placeholder)", + "Neo4j Settings 5.8", + "Neo4j 5.8 (placeholder)", + "Neo4j 5.8" + ); + break; + case V_5_9: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.9 (placeholder)", + "Neo4j Settings 5.9", + "Neo4j 5.9 (placeholder)", + "Neo4j 5.9" + ); + break; + case V_RC: expectedCompatibilities = Set.of( + "Neo4j Settings RC (placeholder)", "Neo4j Settings RC", - "Neo4j Settings DEV (placeholder)", - "Neo4j Settings DEV", - "Neo4j RC", - "Neo4j DEV (placeholder)", - "Neo4j DEV" + "Neo4j RC (placeholder)", + "Neo4j RC" ); break; default: diff --git a/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java b/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java index 7da7024faea..2d03f397403 100644 --- a/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java @@ -321,9 +321,9 @@ default void testRunOnEmptyGraph() { GraphProjectConfig graphProjectConfig = withNameAndProjections( "", loadedGraphName, - NodeProjections.of( + ImmutableNodeProjections.of( Map.of( - NodeLabel.of("X"), NodeProjection.of("X", PropertyMappings.of(propertyMappings)) + NodeLabel.of("X"), ImmutableNodeProjection.of("X", ImmutablePropertyMappings.of(propertyMappings)) ) ), relationshipProjections() diff --git a/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java b/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java index 88df2c89c96..bb5c1618216 100644 --- a/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java @@ -85,7 +85,10 @@ default void testSeedPropertyValidation() { GraphProjectFromStoreConfig graphProjectConfig = ImmutableGraphProjectFromStoreConfig.of( "", graphName, - NodeProjections.single(NodeLabel.of("A"), NodeProjection.of("A", PropertyMappings.of(nodeProperties))), + NodeProjections.single( + NodeLabel.of("A"), + ImmutableNodeProjection.of("A", ImmutablePropertyMappings.of(nodeProperties)) + ), allRelationshipsProjection() ); diff --git a/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java b/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java index e7372764e80..20e908d23e5 100644 --- a/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java +++ b/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java @@ -58,9 +58,9 @@ default GraphProjectFromStoreConfig withNameAndRelationshipProjections( return ImmutableGraphProjectFromStoreConfig.of( userName, graphName, - AbstractNodeProjections.create(singletonMap( + NodeProjections.create(singletonMap( ALL_NODES, - NodeProjection.of(PROJECT_ALL, PropertyMappings.of(propertyMappings)) + ImmutableNodeProjection.of(PROJECT_ALL, ImmutablePropertyMappings.of(propertyMappings)) )), rels ); diff --git a/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java b/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java index a0a4777b810..a1d9d3cd952 100644 --- a/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java @@ -30,7 +30,6 @@ import java.lang.reflect.InvocationTargetException; import java.util.Collections; import java.util.Map; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -60,13 +59,13 @@ default void testWriteBackGraphMutationOnFilteredGraph() { StoreLoaderBuilder storeLoaderBuilder = new StoreLoaderBuilder() .databaseService(graphDb()) .graphName(graphName) - .addNodeProjection(NodeProjection.of( + .addNodeProjection(ImmutableNodeProjection.of( "A", - PropertyMappings.of(nodeProperties().stream().map(PropertyMapping::of).collect(Collectors.toList())) + PropertyMappings.fromObject(nodeProperties()) )) - .addNodeProjection(NodeProjection.of( + .addNodeProjection(ImmutableNodeProjection.of( "B", - PropertyMappings.of(nodeProperties().stream().map(PropertyMapping::of).collect(Collectors.toList())) + PropertyMappings.fromObject(nodeProperties()) )); diff --git a/proc/test/src/main/java/org/neo4j/gds/MutatePropertyProcTest.java b/proc/test/src/main/java/org/neo4j/gds/MutatePropertyProcTest.java index 849b188652d..8a97f2dec50 100644 --- a/proc/test/src/main/java/org/neo4j/gds/MutatePropertyProcTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/MutatePropertyProcTest.java @@ -103,7 +103,7 @@ default void testGraphMutationOnFilteredGraph() { .orElse(Orientation.NATURAL); GraphStore graphStore = new TestNativeGraphLoader(graphDb()) .withLabels("A", "B") - .withNodeProperties(PropertyMappings.of(nodeProperties() + .withNodeProperties(ImmutablePropertyMappings.of(nodeProperties() .stream() .map(PropertyMapping::of) .collect(Collectors.toList()))) diff --git a/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java b/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java index 18d88554f2b..5221a315a06 100644 --- a/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java +++ b/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java @@ -70,7 +70,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long nodeCount() { return 0; } }; diff --git a/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java b/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java new file mode 100644 index 00000000000..ab69529e2bf --- /dev/null +++ b/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.test; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.utils.progress.JobId; +import org.neo4j.gds.core.utils.progress.TaskRegistry; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.core.utils.progress.tasks.Task; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.neo4j.gds.assertj.Extractors.removingThreadId; +import static org.neo4j.gds.assertj.Extractors.replaceTimings; + +@GdlExtension +class ProgressTrackingTest { + + @GdlGraph + static String GDL = + "CREATE " + + " ()-[:REL]->()," + + " ()-[:REL2]->(),"; + + @Inject + Graph graph; + + @Test + void shouldLogProgress() { + var factory = new TestAlgorithmFactory<>(); + var testConfig = TestConfigImpl.builder().logProgress(true).build(); + var log = Neo4jProxy.testLog(); + + factory.build(graph, testConfig, log, TaskRegistryFactory.empty()).compute(); + + assertThat(log.getMessages(TestLog.INFO)) + .extracting(removingThreadId()) + .extracting(replaceTimings()) + .containsExactly( + "TestAlgorithm :: Start", + "TestAlgorithm 50%", + "TestAlgorithm 100%", + "TestAlgorithm :: Finished" + ); + + } + + @Test + void shouldNotLogProgress() { + var factory = new TestAlgorithmFactory<>(); + var testConfig = TestConfigImpl.builder().logProgress(false).build(); + var log = Neo4jProxy.testLog(); + + TaskRegistryFactory taskRegistryFactoryMock = mock(TaskRegistryFactory.class); + TaskRegistry taskRegistryMock = mock(TaskRegistry.class); + doReturn(taskRegistryMock).when(taskRegistryFactoryMock).newInstance(any(JobId.class)); + + factory.build(graph, testConfig, log, taskRegistryFactoryMock).compute(); + + assertThat(log.getMessages(TestLog.INFO)) + .as("When `logProgress` is set to `false` there should only be `start` and `finished` log messages") + .extracting(removingThreadId()) + .extracting(replaceTimings()) + .containsExactly( + "TestAlgorithm :: Start", + "TestAlgorithm :: Finished" + ); + + // Now make sure that the tasks have been registered + verify(taskRegistryMock, times(1)).registerTask(any(Task.class)); + verify(taskRegistryMock, times(1)).unregisterTask(); + } +} diff --git a/settings.gradle b/settings.gradle index ac0c7d7ea73..5f37fd599a6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -130,6 +130,24 @@ project(':neo4j-kernel-adapter-5.1').projectDir = file('compatibility/5.1/neo4j- include('neo4j-kernel-adapter-5.2') project(':neo4j-kernel-adapter-5.2').projectDir = file('compatibility/5.2/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.3') +project(':neo4j-kernel-adapter-5.3').projectDir = file('compatibility/5.3/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.4') +project(':neo4j-kernel-adapter-5.4').projectDir = file('compatibility/5.4/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.5') +project(':neo4j-kernel-adapter-5.5').projectDir = file('compatibility/5.5/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.6') +project(':neo4j-kernel-adapter-5.6').projectDir = file('compatibility/5.6/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.7') +project(':neo4j-kernel-adapter-5.7').projectDir = file('compatibility/5.7/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.8') +project(':neo4j-kernel-adapter-5.8').projectDir = file('compatibility/5.8/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -197,19 +215,37 @@ include('proc-pipeline-catalog') project(':proc-pipeline-catalog').projectDir = file('proc/pipeline-catalog') include('storage-engine-adapter') -project(':storage-engine-adapter').projectDir = file('cypher/common/storage-engine-adapter') +project(':storage-engine-adapter').projectDir = file('compatibility/common/storage-engine-adapter') include('storage-engine-adapter-4.4') -project(':storage-engine-adapter-4.4').projectDir = file('cypher/4.4/storage-engine-adapter') +project(':storage-engine-adapter-4.4').projectDir = file('compatibility/4.4/storage-engine-adapter') include('storage-engine-adapter-5.1') -project(':storage-engine-adapter-5.1').projectDir = file('cypher/5.1/storage-engine-adapter') +project(':storage-engine-adapter-5.1').projectDir = file('compatibility/5.1/storage-engine-adapter') include('storage-engine-adapter-5.2') -project(':storage-engine-adapter-5.2').projectDir = file('cypher/5.2/storage-engine-adapter') +project(':storage-engine-adapter-5.2').projectDir = file('compatibility/5.2/storage-engine-adapter') + +include('storage-engine-adapter-5.3') +project(':storage-engine-adapter-5.3').projectDir = file('compatibility/5.3/storage-engine-adapter') + +include('storage-engine-adapter-5.4') +project(':storage-engine-adapter-5.4').projectDir = file('compatibility/5.4/storage-engine-adapter') + +include('storage-engine-adapter-5.5') +project(':storage-engine-adapter-5.5').projectDir = file('compatibility/5.5/storage-engine-adapter') + +include('storage-engine-adapter-5.6') +project(':storage-engine-adapter-5.6').projectDir = file('compatibility/5.6/storage-engine-adapter') + +include('storage-engine-adapter-5.7') +project(':storage-engine-adapter-5.7').projectDir = file('compatibility/5.7/storage-engine-adapter') + +include('storage-engine-adapter-5.8') +project(':storage-engine-adapter-5.8').projectDir = file('compatibility/5.8/storage-engine-adapter') include('storage-engine-adapter-api') -project(':storage-engine-adapter-api').projectDir = file('cypher/api/storage-engine-adapter') +project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') include('string-formatting') project(':string-formatting').projectDir = file('string-formatting') diff --git a/string-formatting/src/main/java/org/neo4j/gds/utils/StringJoining.java b/string-formatting/src/main/java/org/neo4j/gds/utils/StringJoining.java index 436066925d0..d4d1dbd12af 100644 --- a/string-formatting/src/main/java/org/neo4j/gds/utils/StringJoining.java +++ b/string-formatting/src/main/java/org/neo4j/gds/utils/StringJoining.java @@ -59,7 +59,16 @@ public static String join( CharSequence prefix, CharSequence suffix ) { - return alternatives.stream().sorted().collect(joining(delimiter, prefix, suffix)); + return join(alternatives.stream(), delimiter, prefix, suffix); + } + + public static String join( + Stream alternatives, + CharSequence delimiter, + CharSequence prefix, + CharSequence suffix + ) { + return alternatives.sorted().collect(joining(delimiter, prefix, suffix)); } public static String joinVerbose(Collection alternatives) { diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java index 82be89e2310..0dafb6afcbd 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java @@ -27,14 +27,16 @@ import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.beta.filter.expression.Expression; import org.neo4j.gds.beta.filter.expression.ExpressionParser; import org.neo4j.gds.beta.filter.expression.SemanticErrors; import org.neo4j.gds.beta.filter.expression.ValidationContext; import org.neo4j.gds.config.GraphProjectFromGraphConfig; import org.neo4j.gds.core.loading.GraphStoreBuilder; -import org.neo4j.gds.core.loading.Nodes; +import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -121,7 +123,7 @@ public static GraphStore filter( .databaseId(graphStore.databaseId()) .capabilities(graphStore.capabilities()) .schema(filteredSchema) - .nodes(Nodes.of(filteredNodes.idMap(), filteredNodes.propertyStores())) + .nodes(ImmutableNodes.of(filteredSchema.nodeSchema(), filteredNodes.idMap(), filteredNodes.propertyStores())) .relationshipImportResult(RelationshipImportResult.of(filteredRelationships)) .concurrency(config.concurrency()) .build(); @@ -173,20 +175,22 @@ private static String replaceStarWithTrue(String filter) { return filter.equals(ElementProjection.PROJECT_ALL) ? "true" : filter; } - public static GraphSchema filterSchema(GraphSchema inputGraphSchema, NodesFilter.FilteredNodes filteredNodes, Set filteredRelationshipTypes) { - var nodeSchema = inputGraphSchema.nodeSchema().filter(filteredNodes.idMap().availableNodeLabels()); + public static MutableGraphSchema filterSchema(GraphSchema inputGraphSchema, NodesFilter.FilteredNodes filteredNodes, Set filteredRelationshipTypes) { + var nodeSchema = MutableNodeSchema.from(inputGraphSchema.nodeSchema().filter(filteredNodes.idMap().availableNodeLabels())); if (nodeSchema.availableLabels().isEmpty()) { nodeSchema.addLabel(NodeLabel.ALL_NODES); } - RelationshipSchema relationshipSchema = inputGraphSchema - .relationshipSchema() - .filter(filteredRelationshipTypes); + var relationshipSchema = MutableRelationshipSchema.from( + inputGraphSchema + .relationshipSchema() + .filter(filteredRelationshipTypes) + ); if (relationshipSchema.availableTypes().isEmpty()) { relationshipSchema.addRelationshipType(RelationshipType.ALL_RELATIONSHIPS, Direction.DIRECTED); } - return GraphSchema.of(nodeSchema, relationshipSchema, Map.of()); + return MutableGraphSchema.of(nodeSchema, relationshipSchema, Map.of()); } private GraphStoreFilter() {} diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java index f54713a45e1..7cd225c750d 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java @@ -115,6 +115,7 @@ private static SingleTypeRelationships filterRelationshipType( var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(outputNodes) + .relationshipType(relType) .concurrency(concurrency) .addAllPropertyConfigs(propertyConfigs) .indexInverse(graphStore.inverseIndexedRelationshipTypes().contains(relType)) diff --git a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java index e569c861835..7ce1963ba8b 100644 --- a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java @@ -22,6 +22,7 @@ import org.assertj.core.api.Assertions; import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.Timeout; import org.neo4j.gds.core.Settings; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; @@ -38,6 +39,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -49,6 +51,7 @@ @ImpermanentDbmsExtension(configurationCallback = "configuration") @Neo4jGraphExtension +@Timeout(value = 30, unit = TimeUnit.MINUTES) public abstract class BaseTest { protected static final String DEFAULT_GRAPH_NAME = "graph"; @@ -94,18 +97,18 @@ protected List allNodesWithLabel(String label) { return sourceNodes; } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( @Language("Cypher") String query, Consumer check ) { - QueryRunner.runQueryWithRowConsumer(db, query, check); + return QueryRunner.runQueryWithRowConsumer(db, query, check); } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( @Language("Cypher") String query, BiConsumer check ) { - QueryRunner.runQueryWithRowConsumer(db, query, emptyMap(), check); + return QueryRunner.runQueryWithRowConsumer(db, query, emptyMap(), check); } protected void runQueryWithRowConsumer( @@ -116,29 +119,29 @@ protected void runQueryWithRowConsumer( QueryRunner.runQueryWithRowConsumer(db, query, params, discardTx(check)); } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( GraphDatabaseService localDb, @Language("Cypher") String query, Map params, Consumer check ) { - QueryRunner.runQueryWithRowConsumer(localDb, query, params, discardTx(check)); + return QueryRunner.runQueryWithRowConsumer(localDb, query, params, discardTx(check)); } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( GraphDatabaseService localDb, @Language("Cypher") String query, Consumer check ) { - QueryRunner.runQueryWithRowConsumer(localDb, query, emptyMap(), discardTx(check)); + return QueryRunner.runQueryWithRowConsumer(localDb, query, emptyMap(), discardTx(check)); } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( String username, @Language("Cypher") String query, Consumer check ) { - QueryRunner.runQueryWithRowConsumer(db, username, query, emptyMap(), discardTx(check)); + return QueryRunner.runQueryWithRowConsumer(db, username, query, emptyMap(), discardTx(check)); } protected T runQuery( diff --git a/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java b/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java index b4425b02b2e..f6f40029712 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java +++ b/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java @@ -662,8 +662,8 @@ static GraphProjectFromStoreConfig inlineGraphProjectConfig( .graphName(graphName.orElse("")) .nodeProjections(NodeProjections.create(nodeProjections)) .relationshipProjections(ImmutableRelationshipProjections.builder().putAllProjections(relProjections).build()) - .nodeProperties(PropertyMappings.of(nodeProperties)) - .relationshipProperties(PropertyMappings.of(relProperties)) + .nodeProperties(ImmutablePropertyMappings.of(nodeProperties)) + .relationshipProperties(ImmutablePropertyMappings.of(relProperties)) .build(); } @@ -817,8 +817,8 @@ private static MinimalObject toMinimalObject( ElementProjection projection, ElementIdentifier identifier ) { - if (projection instanceof AbstractNodeProjection) { - return toMinimalObject(((AbstractNodeProjection) projection), identifier); + if (projection instanceof NodeProjection) { + return toMinimalObject(((NodeProjection) projection), identifier); } if (projection instanceof RelationshipProjection) { return toMinimalObject(((RelationshipProjection) projection), identifier); @@ -827,7 +827,7 @@ private static MinimalObject toMinimalObject( } private static MinimalObject toMinimalObject( - AbstractNodeProjection projection, + NodeProjection projection, ElementIdentifier identifier ) { MinimalObject properties = toMinimalObject(projection.properties(), false); @@ -836,12 +836,12 @@ private static MinimalObject toMinimalObject( } Map value = new LinkedHashMap<>(); - value.put(AbstractNodeProjection.LABEL_KEY, projection.label()); + value.put(NodeProjection.LABEL_KEY, projection.label()); properties.toObject().ifPresent(o -> value.put(PROPERTIES_KEY, o)); return MinimalObject.map(value); } - private static boolean matchesLabel(String label, AbstractNodeProjection projection) { + private static boolean matchesLabel(String label, NodeProjection projection) { return Objects.equals(projection.label(), label); } @@ -881,7 +881,7 @@ private static boolean matchesType(String type, RelationshipProjection projectio } private static MinimalObject toMinimalObject( - AbstractPropertyMappings propertyMappings, + PropertyMappings propertyMappings, boolean includeAggregation ) { List mappings = propertyMappings.mappings(); diff --git a/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java b/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java index 9055707b5cd..3ee1d7eb854 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java +++ b/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java @@ -70,7 +70,7 @@ public static GraphProjectFromStoreConfig storeConfig( ) { // Node projections Map tempNP = new LinkedHashMap<>(); - nodeLabels.forEach(label -> tempNP.put(label, NodeProjection.of(label, PropertyMappings.of()))); + nodeLabels.forEach(label -> tempNP.put(label, NodeProjection.of(label))); nodeProjections.forEach(np -> tempNP.put(np.label(), np)); nodeProjectionsWithIdentifier.forEach(tempNP::put); @@ -111,7 +111,7 @@ public static GraphProjectFromStoreConfig storeConfig( .withDefaultAggregation(aggregation) .build(); - NodeProjections np = NodeProjections.of(tempNP.entrySet().stream().collect(Collectors.toMap( + NodeProjections np = ImmutableNodeProjections.of(tempNP.entrySet().stream().collect(Collectors.toMap( e -> NodeLabel.of(e.getKey()), Map.Entry::getValue ))); @@ -126,7 +126,7 @@ public static GraphProjectFromStoreConfig storeConfig( .graphName(graphName.orElse("")) .nodeProjections(np) .relationshipProjections(rp) - .nodeProperties(PropertyMappings.of(nodeProperties)) + .nodeProperties(ImmutablePropertyMappings.of(nodeProperties)) .relationshipProperties(relationshipPropertyMappings) .readConcurrency(concurrency.orElse(ConcurrencyConfig.DEFAULT_CONCURRENCY)) .jobId(jobId.orElse(new JobId())) diff --git a/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java b/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java index 4d83a01e7d1..dfaa7c26aea 100644 --- a/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java +++ b/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds; +import org.apache.commons.lang3.mutable.MutableLong; import org.intellij.lang.annotations.Language; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.graphdb.GraphDatabaseService; @@ -45,53 +46,68 @@ public final class QueryRunner { private QueryRunner() {} - public static void runQueryWithRowConsumer( + public static long runQueryWithRowConsumer( GraphDatabaseService db, String username, @Language("Cypher") String query, Map params, BiConsumer rowConsumer ) { + var rowCounter = new MutableLong(); runInTransaction(db, tx -> { try (KernelTransaction.Revertable ignored = withUsername(tx, username, db.databaseName()); Result result = runQueryWithoutClosingTheResult(tx, query, params)) { result.accept(row -> { rowConsumer.accept(tx, row); + rowCounter.increment(); + return true; }); } }); + + return rowCounter.longValue(); } - public static void runQueryWithRowConsumer( + public static long runQueryWithRowConsumer( GraphDatabaseService db, @Language("Cypher") String query, Map params, BiConsumer rowConsumer ) { + var rowCounter = new MutableLong(); runInTransaction(db, tx -> { try (Result result = runQueryWithoutClosingTheResult(tx, query, params)) { result.accept(row -> { rowConsumer.accept(tx, row); + rowCounter.increment(); + return true; }); } }); + + return rowCounter.longValue(); } - public static void runQueryWithRowConsumer( + public static long runQueryWithRowConsumer( GraphDatabaseService db, @Language("Cypher") String query, Consumer rowConsumer ) { + var rowCounter = new MutableLong(); runInTransaction(db, tx -> { try (Result result = runQueryWithoutClosingTheResult(tx, query, emptyMap())) { result.accept(row -> { rowConsumer.accept(row); + rowCounter.increment(); + return true; }); } }); + + return rowCounter.longValue(); } public static void runQuery(GraphDatabaseService db, @Language("Cypher") String query) { diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index 2b2688b6f77..a736092effe 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -22,7 +22,6 @@ import org.immutables.builder.Builder; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.CSRGraphStoreFactory; import org.neo4j.gds.api.DatabaseId; @@ -31,12 +30,9 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.PropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; import org.neo4j.gds.core.Username; @@ -49,12 +45,13 @@ import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.ImmutablePropertyConfig; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; +import org.neo4j.gds.core.loading.construction.PropertyValues; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; -import org.neo4j.gds.core.loading.nodeproperties.NodePropertiesFromStoreBuilder; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlSupportPerMethodExtension; +import org.neo4j.values.storable.Value; import org.neo4j.values.storable.Values; import org.s1ck.gdl.GDLHandler; import org.s1ck.gdl.model.Element; @@ -158,63 +155,15 @@ public MemoryEstimation estimateMemoryUsageAfterLoading() { return MemoryEstimations.empty(); } - @Override - protected ProgressTracker initProgressTracker() { - return ProgressTracker.NULL_TRACKER; - } - - @Override - protected GraphSchema computeGraphSchema(Nodes nodes, RelationshipImportResult relationshipImportResult) { - var nodeProperties = nodes.properties(); - var nodeSchema = NodeSchema.empty(); - gdlHandler - .getVertices() - .forEach(vertex -> { - var labels = vertex.getLabels().stream().map(NodeLabel::of).collect(Collectors.toList()); - if (labels.isEmpty()) { - labels = List.of(NodeLabel.ALL_NODES); - } - - labels.forEach(label -> vertex - .getProperties() - .forEach((propertyKey, propertyValue) -> nodeSchema - .getOrCreateLabel(label) - .addProperty( - propertyKey, - PropertySchema.of( - propertyKey, - nodeProperties.get(propertyKey).valueType(), - nodeProperties.get(propertyKey).defaultValue(), - nodeProperties.get(propertyKey).propertyState() - ) - ) - )); - }); - // in case there were no properties add all labels - nodes.idMap().availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); - - var relationshipSchema = relationshipImportResult.importResults().entrySet().stream().reduce( - RelationshipSchema.empty(), - (unionSchema, entry) -> { - var relationshipType = entry.getKey(); - var relationships = entry.getValue(); - return unionSchema.union(relationships.relationshipSchema(relationshipType)); - }, - RelationshipSchema::union - ); - - return GraphSchema.of( - nodeSchema, - relationshipSchema, - Map.of() - ); - } - @Override public CSRGraphStore build() { var nodes = loadNodes(); var relationshipImportResult = loadRelationships(nodes.idMap()); - var schema = computeGraphSchema(nodes, relationshipImportResult); + var schema = MutableGraphSchema.of( + nodes.schema(), + relationshipImportResult.relationshipSchema(), + Map.of() + ); return new GraphStoreBuilder() .databaseId(databaseId) @@ -230,7 +179,9 @@ private Nodes loadNodes() { var nodesBuilder = GraphFactory.initNodesBuilder() .maxOriginalId(dimensions.highestPossibleNodeCount() - 1) .hasLabelInformation(true) + .hasProperties(true) .concurrency(1) + .propertyState(graphProjectConfig.propertyState()) .build(); gdlHandler.getVertices().forEach(vertex -> { @@ -241,38 +192,23 @@ private Nodes loadNodes() { .filter(label -> !NodeLabel.ALL_NODES.name().equals(label)) .collect(Collectors.toList()); } - nodesBuilder.addNode( - vertex.getId(), - NodeLabelTokens.of(labels) - ); - }); - var idMap = nodesBuilder.build().idMap(); - - return Nodes.of(idMap, loadNodeProperties(idMap), graphProjectConfig.propertyState()); - } - - private Map loadNodeProperties(IdMap idMap) { - var propertyBuilders = new HashMap(); - - gdlHandler.getVertices().forEach(vertex -> vertex - .getProperties() - .forEach((propertyKey, propertyValue) -> { + Map propertyValues = new HashMap<>(); + vertex.getProperties().forEach((propertyKey, propertyValue) -> { if (propertyValue instanceof List) { propertyValue = convertListProperty((List) propertyValue); } + propertyValues.put(propertyKey, Values.of(propertyValue)); + }); - propertyBuilders.computeIfAbsent(PropertyMapping.of(propertyKey), (key) -> - NodePropertiesFromStoreBuilder.of( - DefaultValue.DEFAULT, - 1 - )).set(vertex.getId(), Values.of(propertyValue)); - })); - - return propertyBuilders - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().build(idMap))); + nodesBuilder.addNode( + vertex.getId(), + NodeLabelTokens.of(labels), + PropertyValues.of(propertyValues) + ); + }); + + return nodesBuilder.build(); } @NotNull @@ -328,7 +264,7 @@ private HashMap> propertyKeysByRelType() { var propertyKeysByRelType = new HashMap>(); Direction orientation = Direction.fromOrientation(graphProjectConfig.orientation()); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); gdlHandler.getEdges().forEach(edge -> { var relType = RelationshipType.of(edge.getLabel()); var entry = relationshipSchema.getOrCreateRelationshipType(relType, orientation); @@ -399,6 +335,7 @@ private Map createRelationshipBuilders( return GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(relTypeAndProperty.getKey()) .orientation(graphProjectConfig.orientation()) .aggregation(graphProjectConfig.aggregation()) .indexInverse(graphProjectConfig.indexInverse()) @@ -428,6 +365,11 @@ private double gdsValue(Element element, String propertyKey, Object gdlValue) { } } + @Override + protected ProgressTracker progressTracker() { + return ProgressTracker.NULL_TRACKER; + } + private static final class GraphDimensionsGdlReader { static GraphDimensions of(GDLHandler gdlHandler) { diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java index 0af1f739c3c..213528a36cb 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java @@ -28,8 +28,8 @@ public final class DoubleArrayTestPropertyValues implements DoubleArrayNodePrope public DoubleArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long size() { - return 0; + public long nodeCount() { + return -1; } @Override diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java index 5088f7f5b4a..fae77987bb1 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java @@ -29,7 +29,7 @@ public final class DoubleTestPropertyValues implements DoubleNodePropertyValues public DoubleTestPropertyValues(LongToDoubleFunction transformer) {this.transformer = transformer;} @Override - public long size() { + public long nodeCount() { return 0; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java index f2f8e723075..b1fc69ad19d 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java @@ -28,8 +28,8 @@ public final class FloatArrayTestPropertyValues implements FloatArrayNodePropert public FloatArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long size() { - return 0; + public long nodeCount() { + return -1; } @Override diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java index 57bf4700de3..ead19184422 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java @@ -28,8 +28,8 @@ public final class LongArrayTestPropertyValues implements LongArrayNodePropertyV public LongArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long size() { - return 0; + public long nodeCount() { + return -1; } @Override diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java index 9ad961cece0..d6e7964781e 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java @@ -28,8 +28,8 @@ public final class LongTestPropertyValues implements LongNodePropertyValues { public LongTestPropertyValues(LongToLongFunction transformer) {this.transformer = transformer;} @Override - public long size() { - return 0; + public long nodeCount() { + return -1; } @Override diff --git a/test-utils/src/test/java/org/neo4j/gds/GraphProjectConfigBuildersTest.java b/test-utils/src/test/java/org/neo4j/gds/GraphProjectConfigBuildersTest.java index 98d918ddfda..d9cd1e4377c 100644 --- a/test-utils/src/test/java/org/neo4j/gds/GraphProjectConfigBuildersTest.java +++ b/test-utils/src/test/java/org/neo4j/gds/GraphProjectConfigBuildersTest.java @@ -58,9 +58,7 @@ static Stream storeConfigs() { Arguments.arguments( new StoreConfigBuilder().jobId(jobId).build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(ALL_NODES, NodeProjection.all()) - .build()) + .nodeProjections(NodeProjections.single(ALL_NODES, NodeProjection.all())) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection(ALL_RELATIONSHIPS, RelationshipProjection.ALL) .build()) @@ -72,9 +70,7 @@ static Stream storeConfigs() { Arguments.arguments( new StoreConfigBuilder().addNodeLabel("Foo").addRelationshipType("BAR").jobId(jobId).build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(NodeLabel.of("Foo"), NodeProjection.of("Foo", PropertyMappings.of())) - .build()) + .nodeProjections(NodeProjections.single(NodeLabel.of("Foo"), NodeProjection.of("Foo"))) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), @@ -93,9 +89,7 @@ static Stream storeConfigs() { .jobId(jobId) .build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(NodeLabel.of("Foo"), NodeProjection.of("Foo", PropertyMappings.of())) - .build()) + .nodeProjections(NodeProjections.single(NodeLabel.of("Foo"), NodeProjection.of("Foo"))) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), @@ -115,9 +109,7 @@ static Stream storeConfigs() { .jobId(jobId) .build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(NodeLabel.of("Foo"), NodeProjection.of("Foo", PropertyMappings.of())) - .build()) + .nodeProjections(NodeProjections.single(NodeLabel.of("Foo"), NodeProjection.of("Foo"))) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), @@ -138,9 +130,7 @@ static Stream storeConfigs() { .jobId(jobId) .build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(NodeLabel.of("Foo"), NodeProjection.of("Foo", PropertyMappings.of())) - .build()) + .nodeProjections(NodeProjections.single(NodeLabel.of("Foo"), NodeProjection.of("Foo"))) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), @@ -168,15 +158,13 @@ static Stream storeConfigs() { .jobId(jobId) .build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection( + .nodeProjections(NodeProjections.single( NodeLabel.of("Foo"), NodeProjection.builder() .label("Foo") .addProperty(PropertyMapping.of("nProp", DefaultValue.of(23.0D))) .build() - ) - .build()) + )) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), diff --git a/test-utils/src/test/java/org/neo4j/gds/gdl/GdlFactoryTest.java b/test-utils/src/test/java/org/neo4j/gds/gdl/GdlFactoryTest.java index bef3c9e5c99..7c4be3c5fea 100644 --- a/test-utils/src/test/java/org/neo4j/gds/gdl/GdlFactoryTest.java +++ b/test-utils/src/test/java/org/neo4j/gds/gdl/GdlFactoryTest.java @@ -32,8 +32,8 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.loading.CollectingConsumer; import java.util.ArrayList; @@ -262,7 +262,7 @@ void correctSchema() { ); var nodeSchema = graph.schema().nodeSchema(); - var expectedNodeSchema = NodeSchema.empty(); + var expectedNodeSchema = MutableNodeSchema.empty(); expectedNodeSchema.getOrCreateLabel(NodeLabel.of("A")) .addProperty("double", ValueType.DOUBLE) @@ -278,7 +278,7 @@ void correctSchema() { assertThat(nodeSchema).isEqualTo(expectedNodeSchema); var relationshipSchema = graph.schema().relationshipSchema(); - var expectedRelationshipSchema = RelationshipSchema.empty(); + var expectedRelationshipSchema = MutableRelationshipSchema.empty(); expectedRelationshipSchema.getOrCreateRelationshipType(RelationshipType.of("A"), Direction.DIRECTED) .addProperty("prop1", ValueType.DOUBLE, PropertyState.PERSISTENT) @@ -315,7 +315,7 @@ void correctRelationshipSchemaDirection(Orientation orientation) { .build() ).build().build(); - var expectedRelationshipSchema = RelationshipSchema.empty(); + var expectedRelationshipSchema = MutableRelationshipSchema.empty(); var direction = Direction.fromOrientation(orientation); expectedRelationshipSchema.getOrCreateRelationshipType(RelationshipType.of("A"), direction)