diff --git a/README.adoc b/README.adoc index 642522587a8..13dc6410389 100644 --- a/README.adoc +++ b/README.adoc @@ -26,9 +26,13 @@ When installing GDS manually, please refer to the below compatibility matrix: .Compatibility matrix (italicized version is in development) |=== |GDS version | Neo4j version | Java Version -.9+<.^|_GDS 2.4.x_ +.12+<.^|_GDS 2.4.x_ +|Neo4j 5.12.0 +.12+.^|Java 17 +|Neo4j 5.11.0 +|Neo4j 5.10.0 +|Neo4j 5.9.0 |Neo4j 5.8.0 -.8+.^|Java 17 |Neo4j 5.7.0 |Neo4j 5.6.0 |Neo4j 5.3.0 @@ -36,7 +40,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.4.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.21 +|Neo4j 4.4.9 - 4.4.26 .1+.^|Java 11 .9+<.^|GDS 2.3.x |Neo4j 5.8.0 @@ -48,7 +52,7 @@ When installing GDS manually, please refer to the below compatibility matrix: |Neo4j 5.3.0 |Neo4j 5.2.0 |Neo4j 5.1.0 -|Neo4j 4.4.9 - 4.4.21 +|Neo4j 4.4.9 - 4.4.23 .1+.^|Java 11 |=== @@ -93,7 +97,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.3.7 + 2.4.6 ---- @@ -105,21 +109,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.7 + 2.4.6 org.neo4j.gds algo - 2.3.7 + 2.4.6 org.neo4j.gds alpha-algo - 2.3.7 + 2.4.6 ---- @@ -131,28 +135,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.7 + 2.4.6 org.neo4j.gds proc - 2.3.7 + 2.4.6 org.neo4j.gds alpha-proc - 2.3.7 + 2.4.6 org.neo4j.gds open-write-services - 2.3.7 + 2.4.6 ---- diff --git a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java index c9c6b94c9df..7fb7529b9c7 100644 --- a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java +++ b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCentralityResultBuilder.java @@ -22,7 +22,7 @@ import org.HdrHistogram.DoubleHistogram; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.api.ProcedureReturnColumns; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.scaling.LogScaler; import org.neo4j.gds.scaling.ScalerFactory; @@ -95,7 +95,7 @@ private Optional computeCentralityHistogram() { return Optional.of(CentralityStatistics.histogram( nodeCount, centralityFunction, - Pools.DEFAULT, + DefaultPool.INSTANCE, concurrency )); } catch (ArrayIndexOutOfBoundsException e) { diff --git a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java index ae782035337..58ee87adef0 100644 --- a/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java +++ b/algo-common/src/main/java/org/neo4j/gds/result/AbstractCommunityResultBuilder.java @@ -22,7 +22,7 @@ import org.HdrHistogram.Histogram; import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.ProcedureReturnColumns; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.ProgressTimer; import java.util.Map; @@ -54,7 +54,7 @@ protected AbstractCommunityResultBuilder( ProcedureReturnColumns returnColumns, int concurrency ) { - this(returnColumns, Pools.DEFAULT, concurrency); + this(returnColumns, DefaultPool.INSTANCE, concurrency); } protected AbstractCommunityResultBuilder( diff --git a/algo-common/src/main/java/org/neo4j/gds/scaling/ScalarScaler.java b/algo-common/src/main/java/org/neo4j/gds/scaling/ScalarScaler.java index 0c003b08c97..d042eb2d099 100644 --- a/algo-common/src/main/java/org/neo4j/gds/scaling/ScalarScaler.java +++ b/algo-common/src/main/java/org/neo4j/gds/scaling/ScalarScaler.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; public abstract class ScalarScaler implements Scaler { @@ -72,13 +71,13 @@ abstract static class AggregatesComputer implements Runnable { private final Partition partition; private final NodePropertyValues properties; private final ProgressTracker progressTracker; - private final AtomicLong nodeCountOmittingMissingProperties; + private long nodeCountOmittingMissingProperties; AggregatesComputer(Partition partition, NodePropertyValues property, ProgressTracker progressTracker) { this.partition = partition; this.properties = property; this.progressTracker = progressTracker; - this.nodeCountOmittingMissingProperties = new AtomicLong(); + this.nodeCountOmittingMissingProperties = 0L; } abstract void compute(double propertyValue); @@ -89,7 +88,7 @@ public void run() { for (long nodeId = partition.startNode(); nodeId < end; nodeId++) { var propertyValue = properties.doubleValue(nodeId); if (!Double.isNaN(propertyValue)) { - nodeCountOmittingMissingProperties.incrementAndGet(); + ++nodeCountOmittingMissingProperties; compute(propertyValue); } } @@ -97,7 +96,7 @@ public void run() { } long nodeCountOmittingMissingValues() { - return nodeCountOmittingMissingProperties.get(); + return nodeCountOmittingMissingProperties; } } } diff --git a/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java b/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java index ff833682001..5063bad3c1f 100644 --- a/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/result/CommunityStatisticsTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.TestSupport; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import java.util.HashMap; import java.util.Map; @@ -49,7 +49,7 @@ void communitySizes( var communitySizes = CommunityStatistics.communitySizes( nodeCount, communityFunction, - Pools.DEFAULT, + DefaultPool.INSTANCE, concurrency ); expectedCommunitySizes.forEach((communityId, expectedSize) -> { @@ -72,7 +72,7 @@ void communityCount( CommunityStatistics.communityCount( nodeCount, communityFunction, - Pools.DEFAULT, + DefaultPool.INSTANCE, concurrency ) ); @@ -91,7 +91,7 @@ void communityCountAndHistogram( var communityCountAndHistogram = CommunityStatistics.communityCountAndHistogram( nodeCount, communityFunction, - Pools.DEFAULT, + DefaultPool.INSTANCE, concurrency ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java index 24159e087e6..20ad760ed70 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/CenterTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -63,7 +63,7 @@ void normalizes(NodePropertyValues properties, double avg, double[] expected) { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.avg).isEqualTo(avg); @@ -81,7 +81,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var avg = 4.444; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java index 1acf05847a2..10b7cef7416 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/L1NormTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -62,7 +62,7 @@ void scale(int nodeCount, NodePropertyValues properties, double l1norm, double[] nodeCount, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.l1Norm).isEqualTo(l1norm); @@ -79,7 +79,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); for (int i = 0; i < 10; i++) { @@ -95,7 +95,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var l1norm = 40D; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java index 0808221440a..b8f93a15f2f 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/L2NormTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -53,7 +53,7 @@ void normalizes(NodePropertyValues properties, double euclideanLength, double[] 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.euclideanLength).isEqualTo(euclideanLength); @@ -70,7 +70,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); for (int i = 0; i < 10; i++) { @@ -86,7 +86,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var euclideanLength = 16.124; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java index 220eebde88f..680bfd05608 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/LogScalerTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; import org.neo4j.gds.nodeproperties.LongTestPropertyValues; @@ -72,7 +72,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); for (int i = 0; i < 5; i++) { diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java index ba805420c8e..9d50bbb5afc 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MaxTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -70,7 +70,7 @@ void scale(int nodeCount, NodePropertyValues properties, double absMax, double[] nodeCount, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.maxAbs).isEqualTo(absMax); @@ -88,7 +88,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.statistics()).containsExactlyEntriesOf(Map.of("absMax", List.of(0D))); @@ -106,7 +106,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java index 80ac7bc12ab..39b5a4bf82c 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MeanTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -57,7 +57,7 @@ void normalizes(NodePropertyValues properties, double avg, double min, double ma 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.avg).isEqualTo(avg); @@ -81,7 +81,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.statistics()).containsExactlyInAnyOrderEntriesOf(Map.of( @@ -103,7 +103,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var avg = 4.444; diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java index d2f0fe13f9a..edc881c051d 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/MinMaxTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -53,7 +53,7 @@ void normalizes(NodePropertyValues properties, double min, double max) { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.min).isEqualTo(min); @@ -77,7 +77,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.statistics()).containsExactlyEntriesOf(Map.of( @@ -98,7 +98,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java index c67bc368df1..a5f6c2966ab 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/ScalerTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -39,14 +39,14 @@ void shouldAccumulateStatsCorrectly() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var meanScaler2 = (Mean) Mean.buildFrom(CypherMapWrapper.empty()).create( new DoubleTestPropertyValues(nodeId -> 2 * nodeId), 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var meanScaler1Stats = meanScaler1.statistics(); diff --git a/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java b/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java index 8495767c73b..236975d31ff 100644 --- a/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java +++ b/algo-common/src/test/java/org/neo4j/gds/scaling/StdScoreTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.nodeproperties.DoubleTestPropertyValues; @@ -56,7 +56,7 @@ void normalizes(NodePropertyValues properties, double avg, double std, double[] 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(scaler.avg).isEqualTo(avg); @@ -78,7 +78,7 @@ void handlesMissingValue() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var avg = 4.444; @@ -100,7 +100,7 @@ void avoidsDivByZero() { 10, 1, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); for (int i = 0; i < 10; i++) { diff --git a/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java b/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java index 31fd6d65d44..314e898be44 100644 --- a/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java +++ b/algo-test/src/main/java/org/neo4j/gds/test/TestMutateConfig.java @@ -21,18 +21,18 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface TestMutateConfig extends TestConfig, MutatePropertyConfig { +public interface TestMutateConfig extends TestConfig, MutateNodePropertyConfig { default boolean throwOnEstimate() { return false; - }; - + } + static TestMutateConfig of(CypherMapWrapper config) { return new TestMutateConfigImpl(config); } diff --git a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java index c8e6a41d9bf..617846da09a 100644 --- a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java +++ b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutFactory.java @@ -24,7 +24,7 @@ import org.neo4j.gds.approxmaxkcut.config.ApproxMaxKCutConfig; import org.neo4j.gds.collections.haa.HugeAtomicByteArray; import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeByteArray; @@ -51,7 +51,7 @@ public ApproxMaxKCut build( CONFIG configuration, ProgressTracker progressTracker ) { - return new ApproxMaxKCut(graph, Pools.DEFAULT, configuration, progressTracker); + return new ApproxMaxKCut(graph, DefaultPool.INSTANCE, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/config/ApproxMaxKCutMutateConfig.java b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/config/ApproxMaxKCutMutateConfig.java index 3c7856f4726..2eb2f468528 100644 --- a/algo/src/main/java/org/neo4j/gds/approxmaxkcut/config/ApproxMaxKCutMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/approxmaxkcut/config/ApproxMaxKCutMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface ApproxMaxKCutMutateConfig extends ApproxMaxKCutConfig, MutatePropertyConfig { +public interface ApproxMaxKCutMutateConfig extends ApproxMaxKCutConfig, MutateNodePropertyConfig { static ApproxMaxKCutMutateConfig of(CypherMapWrapper config) { return new ApproxMaxKCutMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java index 7925fc0cb21..9bae0ded336 100644 --- a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -66,7 +66,7 @@ public BetweennessCentrality build( graph, strategy, traverserFactory, - Pools.DEFAULT, + DefaultPool.INSTANCE, configuration.concurrency(), progressTracker ); diff --git a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityMutateConfig.java index 981f361ad42..b482d3bedd4 100644 --- a/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/betweenness/BetweennessCentralityMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface BetweennessCentralityMutateConfig extends BetweennessCentralityBaseConfig, MutatePropertyConfig { +public interface BetweennessCentralityMutateConfig extends BetweennessCentralityBaseConfig, MutateNodePropertyConfig { static BetweennessCentralityMutateConfig of(CypherMapWrapper config) { return new BetweennessCentralityMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java index c2219971cc2..df3bd6b0353 100644 --- a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -37,7 +37,7 @@ public ClosenessCentrality build( return ClosenessCentrality.of( graph, configuration, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityMutateConfig.java index b1ffb92afeb..2e235b96446 100644 --- a/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityMutateConfig.java @@ -21,11 +21,11 @@ import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface ClosenessCentralityMutateConfig extends ClosenessCentralityConfig, MutatePropertyConfig { +public interface ClosenessCentralityMutateConfig extends ClosenessCentralityConfig, MutateNodePropertyConfig { static ClosenessCentralityMutateConfig of(CypherMapWrapper config) { return new ClosenessCentralityMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java b/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java index ace1f7357e5..a7d1050f318 100644 --- a/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java +++ b/algo/src/main/java/org/neo4j/gds/conductance/ConductanceFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -43,7 +43,7 @@ public Conductance build( CONFIG configuration, ProgressTracker progressTracker ) { - return new Conductance(graph, Pools.DEFAULT, configuration, progressTracker); + return new Conductance(graph, DefaultPool.INSTANCE, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java index 150061c5256..c3387586ad4 100644 --- a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityFactory.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -45,7 +45,7 @@ public DegreeCentrality build( CONFIG configuration, ProgressTracker progressTracker ) { - return new DegreeCentrality(graph, Pools.DEFAULT, configuration, progressTracker); + return new DegreeCentrality(graph, DefaultPool.INSTANCE, configuration, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityMutateConfig.java index c74bb0d029b..1dc9ae7b10b 100644 --- a/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/degree/DegreeCentralityMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface DegreeCentralityMutateConfig extends DegreeCentralityConfig, MutatePropertyConfig { +public interface DegreeCentralityMutateConfig extends DegreeCentralityConfig, MutateNodePropertyConfig { static DegreeCentralityMutateConfig of(CypherMapWrapper config) { return new DegreeCentralityMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java index 34234604985..14c1e4375e9 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRP.java @@ -22,8 +22,8 @@ import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -221,7 +221,7 @@ void propagateEmbeddings() { ); ParallelUtil.parallelPartitionsConsume( - RunWithConcurrency.builder().executor(Pools.DEFAULT).concurrency(concurrency), + RunWithConcurrency.builder().executor(DefaultPool.INSTANCE).concurrency(concurrency), partitions.stream(), taskSupplier ); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPMutateConfig.java b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPMutateConfig.java index 96a22efff00..2cec7056641 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface FastRPMutateConfig extends FastRPBaseConfig, MutatePropertyConfig { +public interface FastRPMutateConfig extends FastRPBaseConfig, MutateNodePropertyConfig { static FastRPMutateConfig of(CypherMapWrapper userInput) { return new FastRPMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java index 8d1736e0f40..f47eb2e3599 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.config.MutateConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -49,7 +49,7 @@ public GraphSageAlgorithmFactory(ModelCatalog modelCatalog) { @Override public GraphSage build(GraphStore graphStore, CONFIG configuration, ProgressTracker progressTracker) { - var executorService = Pools.DEFAULT; + var executorService = DefaultPool.INSTANCE; var model = resolveModel( modelCatalog, configuration.modelUser(), diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageMutateConfig.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageMutateConfig.java index 56c931feef4..23e10f6f4ff 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface GraphSageMutateConfig extends GraphSageBaseConfig, MutatePropertyConfig { +public interface GraphSageMutateConfig extends GraphSageBaseConfig, MutateNodePropertyConfig { long serialVersionUID = 0x42L; diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java index d8ef87ba90c..358316dbe2c 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -58,7 +58,7 @@ public GraphSageTrain build( GraphSageTrainConfig configuration, ProgressTracker progressTracker ) { - var executorService = Pools.DEFAULT; + var executorService = DefaultPool.INSTANCE; if(configuration.hasRelationshipWeightProperty()) { validateRelationshipWeightPropertyValue(graph, configuration.concurrency(), executorService); } diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNMutateConfig.java b/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNMutateConfig.java index 3eb33296b00..425f687a1e2 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNMutateConfig.java @@ -20,11 +20,11 @@ package org.neo4j.gds.embeddings.hashgnn; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface HashGNNMutateConfig extends HashGNNConfig, MutatePropertyConfig { +public interface HashGNNMutateConfig extends HashGNNConfig, MutateNodePropertyConfig { static HashGNNMutateConfig of(CypherMapWrapper config) { return new HashGNNMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java index 4b78949d26f..48f7d1cba55 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2Vec.java @@ -21,7 +21,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeObjectArray; @@ -60,7 +60,7 @@ public Node2VecModel.Result compute() { graph, config, progressTracker, - Pools.DEFAULT + DefaultPool.INSTANCE ); var probabilitiesBuilder = new RandomWalkProbabilities.Builder( diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecMutateConfig.java b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecMutateConfig.java index 6a75985f6d0..712926bea91 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface Node2VecMutateConfig extends Node2VecBaseConfig, MutatePropertyConfig { +public interface Node2VecMutateConfig extends Node2VecBaseConfig, MutateNodePropertyConfig { static Node2VecMutateConfig of(CypherMapWrapper userInput) { return new Node2VecMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java index 82ed47682a0..879f6fa54bd 100644 --- a/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/harmonic/HarmonicCentralityAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; public class HarmonicCentralityAlgorithmFactory extends GraphAlgorithmFactory { @@ -39,7 +39,7 @@ public HarmonicCentrality build( return new HarmonicCentrality( graph, configuration.concurrency(), - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java index 8430d1e0f92..e1a9eb96a16 100644 --- a/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/indexInverse/InverseRelationshipsAlgorithmFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.AdjacencyListBehavior; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -45,7 +45,7 @@ public InverseRelationships build( InverseRelationshipsConfig configuration, ProgressTracker progressTracker ) { - return new InverseRelationships(graphStore, configuration, progressTracker, Pools.DEFAULT); + return new InverseRelationships(graphStore, configuration, progressTracker, DefaultPool.INSTANCE); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java index ffccd82a491..4cedaee5b3f 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java @@ -31,8 +31,6 @@ import java.util.Optional; import java.util.concurrent.ExecutorService; -import java.util.stream.LongStream; -import java.util.stream.Stream; public class CELF extends Algorithm { @@ -185,12 +183,4 @@ private void lazyForwardPart(long firstSeedNode) { } progressTracker.endSubTask(); } - - public Stream resultStream() { - return LongStream.of(seedSetNodes.keys().toArray()) - .mapToObj(node -> new InfluenceMaximizationResult( - graph.toOriginalNodeId(node), - seedSetNodes.getOrDefault(node, 0) - )); - } } diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java index 69e3692a5b3..f479f7d5d6d 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELFAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -53,7 +53,7 @@ public CELF build( configuration.seedSetSize(), configuration.propagationProbability(), configuration.monteCarloSimulations(), - Pools.DEFAULT, + DefaultPool.INSTANCE, configuration.concurrency(), configuration.randomSeed().orElse(0L), DEFAULT_BATCH_SIZE, diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationMutateConfig.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationMutateConfig.java index 97c901db9d0..d8b1f85df13 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationMutateConfig.java @@ -20,11 +20,11 @@ package org.neo4j.gds.influenceMaximization; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface InfluenceMaximizationMutateConfig extends InfluenceMaximizationBaseConfig, MutatePropertyConfig { +public interface InfluenceMaximizationMutateConfig extends InfluenceMaximizationBaseConfig, MutateNodePropertyConfig { static InfluenceMaximizationMutateConfig of(CypherMapWrapper userInput) { return new InfluenceMaximizationMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java index 1f532f1c713..2e4e0694855 100644 --- a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java +++ b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.config.BaseConfig; import org.neo4j.gds.config.IterationsConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -54,7 +54,7 @@ public K1Coloring build( configuration.maxIterations(), configuration.batchSize(), configuration.concurrency(), - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringMutateConfig.java b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringMutateConfig.java index 519a7ca256c..38330a18072 100644 --- a/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/k1coloring/K1ColoringMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface K1ColoringMutateConfig extends K1ColoringConfig, MutatePropertyConfig { +public interface K1ColoringMutateConfig extends K1ColoringConfig, MutateNodePropertyConfig { static K1ColoringMutateConfig of(CypherMapWrapper userInput) { return new K1ColoringMutateConfigImpl(userInput); } diff --git a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecomposition.java b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecomposition.java index c3da74dcfbd..cde9cf42b1e 100644 --- a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecomposition.java +++ b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecomposition.java @@ -116,6 +116,7 @@ public KCoreDecompositionResult compute() { for (var task : tasks) { task.setScanningDegree(scanningDegree); + task.setPhase(KCoreDecompositionTask.KCoreDecompositionPhase.SCAN); } RunWithConcurrency.builder().tasks(tasks).concurrency(concurrency).run(); @@ -129,8 +130,14 @@ public KCoreDecompositionResult compute() { if (nextScanningDegree == scanningDegree) { degeneracy = scanningDegree; + + for (var task : tasks) { + task.setPhase(KCoreDecompositionTask.KCoreDecompositionPhase.ACT); + } + RunWithConcurrency.builder().tasks(tasks).concurrency(concurrency).run(); scanningDegree++; + } else { //this is a minor optimization not in paper: // if we do not do any updates this round, let's skip directly to the smallest active degree remaining diff --git a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionMutateConfig.java b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionMutateConfig.java index bcd8ea2aadb..584973a1c9b 100644 --- a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionMutateConfig.java @@ -20,11 +20,11 @@ package org.neo4j.gds.kcore; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface KCoreDecompositionMutateConfig extends KCoreDecompositionBaseConfig, MutatePropertyConfig { +public interface KCoreDecompositionMutateConfig extends KCoreDecompositionBaseConfig, MutateNodePropertyConfig { static KCoreDecompositionMutateConfig of(CypherMapWrapper userInput) { return new KCoreDecompositionMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionTask.java b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionTask.java index 3f5f9dacb22..854b08dfe93 100644 --- a/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionTask.java +++ b/algo/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionTask.java @@ -80,13 +80,15 @@ void updateNodeProvider(NodeProvider nodeProvider) { public void run() { if (phase == KCoreDecompositionPhase.SCAN) { scan(); - phase = KCoreDecompositionPhase.ACT; } else { act(); - phase = KCoreDecompositionPhase.SCAN; } } + void setPhase(KCoreDecompositionPhase phase) { + this.phase = phase; + } + private void scan() { long upperBound = nodeProvider.size(); diff --git a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java index b1f5d26521d..25aab9a298f 100644 --- a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -61,7 +61,7 @@ public Kmeans build( return Kmeans.createKmeans(graph, configuration, ImmutableKmeansContext .builder() .progressTracker(progressTracker) - .executor(Pools.DEFAULT) + .executor(DefaultPool.INSTANCE) .build()); } diff --git a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java index d90627dfefb..b5729e27c47 100644 --- a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java +++ b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansContext.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.concurrent.ExecutorService; @@ -31,7 +31,7 @@ public interface KmeansContext { @Value.Default default ExecutorService executor() { - return Pools.DEFAULT; + return DefaultPool.INSTANCE; } @Value.Default diff --git a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansMutateConfig.java b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansMutateConfig.java index 04eb8a32a68..e77279582ec 100644 --- a/algo/src/main/java/org/neo4j/gds/kmeans/KmeansMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/kmeans/KmeansMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface KmeansMutateConfig extends KmeansBaseConfig, MutatePropertyConfig { +public interface KmeansMutateConfig extends KmeansBaseConfig, MutateNodePropertyConfig { static KmeansMutateConfig of(CypherMapWrapper userInput) { return new KmeansMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java b/algo/src/main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java index dcdbd4f4650..13560970dd3 100644 --- a/algo/src/main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java +++ b/algo/src/main/java/org/neo4j/gds/kspanningtree/KSpanningTree.java @@ -58,7 +58,7 @@ public KSpanningTree( super(progressTracker); this.graph = graph; this.minMax = minMax; - this.startNodeId = (int) graph.toMappedNodeId(startNodeId); + this.startNodeId = startNodeId; this.k = k; } diff --git a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java index 33c1a0db598..08232a8ab0f 100644 --- a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java +++ b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.LongDoubleScatterMap; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -53,7 +53,7 @@ public LabelPropagation build( return new LabelPropagation( graph, configuration, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateConfig.java b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateConfig.java index a5a910e60c0..5cc34e56d6b 100644 --- a/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/labelpropagation/LabelPropagationMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface LabelPropagationMutateConfig extends LabelPropagationBaseConfig, MutatePropertyConfig { +public interface LabelPropagationMutateConfig extends LabelPropagationBaseConfig, MutateNodePropertyConfig { static LabelPropagationMutateConfig of(CypherMapWrapper userInput) { return new LabelPropagationMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java index 5fd0d037076..b9b27794150 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java @@ -25,7 +25,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -77,7 +77,7 @@ public Leiden( this.theta = theta; this.randomSeed = randomSeed; // TODO: Pass these two as parameters - this.executorService = Pools.DEFAULT; + this.executorService = DefaultPool.INSTANCE; this.concurrency = concurrency; this.dendrogramManager = new LeidenDendrogramManager( rootGraph, diff --git a/algo/src/main/java/org/neo4j/gds/leiden/LeidenMutateConfig.java b/algo/src/main/java/org/neo4j/gds/leiden/LeidenMutateConfig.java index 0f14102d0f2..0ee812a5342 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/LeidenMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/LeidenMutateConfig.java @@ -20,11 +20,11 @@ package org.neo4j.gds.leiden; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration -public interface LeidenMutateConfig extends LeidenBaseConfig, MutatePropertyConfig { +public interface LeidenMutateConfig extends LeidenBaseConfig, MutateNodePropertyConfig { static LeidenMutateConfig of(CypherMapWrapper userInput) { return new LeidenMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java index 9898c9e3db5..7d8e79dda4f 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/LouvainAlgorithmFactory.java @@ -31,7 +31,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.NativeFactory; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -61,7 +61,7 @@ public Louvain build( configuration.tolerance(), configuration.concurrency(), progressTracker, - Pools.DEFAULT + DefaultPool.INSTANCE ); } diff --git a/algo/src/main/java/org/neo4j/gds/louvain/LouvainMutateConfig.java b/algo/src/main/java/org/neo4j/gds/louvain/LouvainMutateConfig.java index 3ee19b17901..0b8d575869e 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/LouvainMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/LouvainMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface LouvainMutateConfig extends LouvainBaseConfig, MutatePropertyConfig { +public interface LouvainMutateConfig extends LouvainBaseConfig, MutateNodePropertyConfig { static LouvainMutateConfig of(CypherMapWrapper userInput) { return new LouvainMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java index 17fa061e0ed..e3047b883c9 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationFactory.java @@ -25,7 +25,7 @@ import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; import org.neo4j.gds.config.BaseConfig; import org.neo4j.gds.config.IterationsConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -106,7 +106,7 @@ public ModularityOptimization build( seedProperty, configuration.concurrency(), configuration.batchSize(), - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateConfig.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateConfig.java index 9e0ac155dcc..e74d936d5ee 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface ModularityOptimizationMutateConfig extends ModularityOptimizationConfig, MutatePropertyConfig { +public interface ModularityOptimizationMutateConfig extends ModularityOptimizationConfig, MutateNodePropertyConfig { static ModularityOptimizationMutateConfig of(CypherMapWrapper userInput) { return new ModularityOptimizationMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java index e076c0856e5..6ead19c2b5c 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankAlgorithmFactory.java @@ -27,8 +27,8 @@ import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelComputation; import org.neo4j.gds.beta.pregel.PregelSchema; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -44,9 +44,6 @@ public class PageRankAlgorithmFactory extends GraphAlgorithmFactory { - static Task pagerankProgressTask(Graph graph, CONFIG config) { - return Pregel.progressTask(graph, config, "PageRank"); - } private static double averageDegree(Graph graph, int concurrency) { var degreeSum = new LongAdder(); @@ -60,9 +57,19 @@ private static double averageDegree(Graph graph, int concurrency) { } public enum Mode { - PAGE_RANK, - ARTICLE_RANK, - EIGENVECTOR, + PAGE_RANK("PageRank"), + ARTICLE_RANK("ArticleRank"), + EIGENVECTOR("EigenVector"); + + private final String taskName; + + Mode(String taskName) { + this.taskName = taskName; + } + + String taskName() { + return taskName; + } } private final Mode mode; @@ -77,7 +84,7 @@ public PageRankAlgorithmFactory(Mode mode) { @Override public String taskName() { - return mode.name(); + return mode.taskName(); } @Override @@ -121,14 +128,14 @@ public PageRankAlgorithm build( configuration, computation, mode, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } @Override public Task progressTask(Graph graph, CONFIG config) { - return pagerankProgressTask(graph, config); + return Pregel.progressTask(graph, config, taskName()); } @NotNull @@ -143,7 +150,7 @@ private LongToDoubleFunction degreeFunction( var degreeCentrality = new DegreeCentrality( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankMutateConfig.java b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankMutateConfig.java index 4179283b83d..8e4714eed95 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/PageRankMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/PageRankMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface PageRankMutateConfig extends PageRankConfig, MutatePropertyConfig { +public interface PageRankMutateConfig extends PageRankConfig, MutateNodePropertyConfig { static PageRankMutateConfig of(CypherMapWrapper userInput) { return new PageRankMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java index b7d38aba171..23456c2dd14 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java +++ b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaSteppingFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -38,7 +38,7 @@ public DeltaStepping build( T configuration, ProgressTracker progressTracker ) { - return DeltaStepping.of(graph, configuration, Pools.DEFAULT, progressTracker); + return DeltaStepping.of(graph, configuration, DefaultPool.INSTANCE, progressTracker); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java b/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java index 4f21860c24a..15ea38e5e36 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java +++ b/algo/src/main/java/org/neo4j/gds/paths/traverse/BFS.java @@ -23,8 +23,8 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicLongArray; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -199,7 +199,7 @@ public HugeLongArray compute() { if (currentDepth == maximumDepth) { break; } - ParallelUtil.run(bfsTaskList, Pools.DEFAULT); + ParallelUtil.run(bfsTaskList, DefaultPool.INSTANCE); if (targetFoundIndex.get() != Long.MAX_VALUE) { break; diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index 7eb0cf73b97..a07aa9bfcee 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -21,7 +21,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -113,7 +113,7 @@ public PathFindingResult compute() { RunWithConcurrency.builder() .concurrency(config.concurrency()) .tasks(tasks) - .executor(Pools.DEFAULT) + .executor(DefaultPool.INSTANCE) .run(); progressTracker.logProgress(); diff --git a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java index 99f7dbb2498..745c32c2a85 100644 --- a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java +++ b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -69,7 +69,7 @@ public ScaleProperties build( graph, configuration, progressTracker, - Pools.DEFAULT + DefaultPool.INSTANCE ); } diff --git a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesMutateConfig.java b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesMutateConfig.java index d71b9192c01..cb55d86186f 100644 --- a/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/scaleproperties/ScalePropertiesMutateConfig.java @@ -20,12 +20,12 @@ package org.neo4j.gds.scaleproperties; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration @SuppressWarnings("immutables:subtype") -public interface ScalePropertiesMutateConfig extends ScalePropertiesBaseConfig, MutatePropertyConfig { +public interface ScalePropertiesMutateConfig extends ScalePropertiesBaseConfig, MutateNodePropertyConfig { static ScalePropertiesMutateConfig of(CypherMapWrapper userInput) { return new ScalePropertiesMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java index ef6dbe66a79..056fac86cbf 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnFactory.java @@ -23,7 +23,7 @@ import org.apache.commons.lang3.function.TriFunction; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -68,7 +68,7 @@ public FilteredKnn build(Graph graph, CONFIG configuration, ProgressTracker prog KnnContext knnContext = ImmutableKnnContext .builder() .progressTracker(progressTracker) - .executor(Pools.DEFAULT) + .executor(DefaultPool.INSTANCE) .build(); if (configuration.seedTargetNodes()) { diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java index 2ed1e183ead..d94c0992432 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnMutateConfig.java @@ -21,14 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface FilteredKnnMutateConfig extends FilteredKnnBaseConfig, MutatePropertyConfig, MutateRelationshipConfig { +public interface FilteredKnnMutateConfig extends FilteredKnnBaseConfig, + MutateRelationshipPropertyConfig, MutateRelationshipConfig { static FilteredKnnMutateConfig of(CypherMapWrapper config) { return new FilteredKnnMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java index 9e4940c9143..42eac124cb9 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.BitSet; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -59,7 +59,7 @@ public NodeSimilarity build( configuration.sourceNodeFilter().toNodeFilter(graph), configuration.targetNodeFilter().toNodeFilter(graph), configuration.concurrency(), - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java index cab6b05a4f7..54acdbc5726 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnContext.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.concurrent.ExecutorService; @@ -31,7 +31,7 @@ public interface KnnContext { @Value.Default default ExecutorService executor() { - return Pools.DEFAULT; + return DefaultPool.INSTANCE; } @Value.Default diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java index d09bff5787d..8dded7f2ec8 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.LongArrayList; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -59,7 +59,7 @@ public Knn build( ImmutableKnnContext .builder() .progressTracker(progressTracker) - .executor(Pools.DEFAULT) + .executor(DefaultPool.INSTANCE) .build() ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java index 8cf2746da65..39b7c9d277e 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateConfig.java @@ -21,14 +21,14 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface KnnMutateConfig extends KnnBaseConfig, MutatePropertyConfig, MutateRelationshipConfig { +public interface KnnMutateConfig extends KnnBaseConfig, MutateRelationshipPropertyConfig, MutateRelationshipConfig { static KnnMutateConfig of(CypherMapWrapper config) { return new KnnMutateConfigImpl(config); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/Euclidean.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/Euclidean.java index 39c85fbaaea..48524b2632d 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/Euclidean.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/Euclidean.java @@ -19,6 +19,8 @@ */ package org.neo4j.gds.similarity.knn.metrics; +import java.util.function.IntToDoubleFunction; + /** * Here we calculate Euclidean similarity metrics using Euclidean dictance as described in e.g. * https://en.wikipedia.org/wiki/Euclidean_distance @@ -33,22 +35,28 @@ public final class Euclidean { private Euclidean() {} public static double floatMetric(float[] left, float[] right) { - var len = Math.min(left.length, right.length); - var result = 0D; - for (int i = 0; i < len; i++) { - double delta = left[i] - right[i]; - result += delta * delta; - } - return 1.0 / (1.0 + result); + return compute( + Math.min(left.length, right.length), + i -> left[i], + i -> right[i] + ); } public static double doubleMetric(double[] left, double[] right) { - var len = Math.min(left.length, right.length); + return compute( + Math.min(left.length, right.length), + i -> left[i], + i -> right[i] + ); + } + + private static double compute(int len, IntToDoubleFunction left, IntToDoubleFunction right) { var result = 0D; for (int i = 0; i < len; i++) { - double delta = left[i] - right[i]; + double delta = left.applyAsDouble(i) - right.applyAsDouble(i); result += delta * delta; } - return 1.0 / (1.0 + result); + + return 1.0 / (1.0 + Math.sqrt(result)); } } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputer.java new file mode 100644 index 00000000000..2279547d23a --- /dev/null +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputer.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.nodesim; + +import org.neo4j.gds.core.utils.Intersections; + +public class CosineSimilarityComputer implements MetricSimilarityComputer { + private final double similarityCutoff; + + public CosineSimilarityComputer(double similarityCutoff) { + this.similarityCutoff = similarityCutoff; + } + + @Override + public double computeSimilarity(long[] vector1, long[] vector2) { + var intersection = Intersections.intersection3(vector1, vector2); + var similarity = intersection / (Math.sqrt(vector1.length) * Math.sqrt(vector2.length)); + return similarity >= similarityCutoff ? similarity : Double.NaN; + } + + @Override + public double computeWeightedSimilarity(long[] vector1, long[] vector2, double[] weights1, double[] weights2) { + assert vector1.length == weights1.length; + assert vector2.length == weights2.length; + + double vector1SquaredSum = 0; + double vector2SquaredSum = 0; + double above=0; + + int offset1 = 0; + int offset2 = 0; + int length1 = weights1.length; + int length2 = weights2.length; + + while (offset1 < length1 && offset2 < length2) { + long target1 = vector1[offset1]; + long target2 = vector2[offset2]; + double w1 = weights1[offset1]; + double w2 = weights2[offset2]; + + if (target1 == target2) { + above+=w1*w2; + vector1SquaredSum+= w1*w1; + vector2SquaredSum+= w2*w2; + offset1++; + offset2++; + } else if (target1 < target2) { + vector1SquaredSum += w1*w1; + offset1++; + } else { + vector2SquaredSum += w2*w2; + offset2++; + } + } + + for (; offset1 < length1; offset1++) { + vector1SquaredSum += weights1[offset1] * weights1[offset1]; + } + for (; offset2 < length2; offset2++) { + vector2SquaredSum += weights2[offset2]* weights2[offset2]; + } + + + double similarity = above/(Math.sqrt(vector1SquaredSum) * Math.sqrt(vector2SquaredSum)); + return similarity >= similarityCutoff ? similarity : Double.NaN; + + } + + static class Builder implements MetricSimilarityComputerBuilder { + public MetricSimilarityComputer build(double similarityCutoff) { + return new CosineSimilarityComputer(similarityCutoff); + } + + @Override + public String render() { + return "COSINE"; + } + } +} diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputer.java index a1d2626385f..191491f4cfc 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputer.java @@ -50,6 +50,8 @@ private static NodeSimilarityMetric valueOf(String userInput) { return NodeSimilarityMetric.JACCARD; } else if (userInputInCaps.equals("OVERLAP")) { return NodeSimilarityMetric.OVERLAP; + } else if (userInputInCaps.equals("COSINE")) { + return NodeSimilarityMetric.COSINE; } else { throw new IllegalArgumentException(userInput + " is not a valid metric. Available metrics include Jaccard and Overlap"); } @@ -58,8 +60,10 @@ private static NodeSimilarityMetric valueOf(String userInput) { private static MetricSimilarityComputerBuilder create(NodeSimilarityMetric metric) { if (metric == NodeSimilarityMetric.JACCARD) { return new JaccardSimilarityComputer.Builder(); - } else { + } else if (metric == NodeSimilarityMetric.OVERLAP) { return new OverlapSimilarityComputer.Builder(); + } else { + return new CosineSimilarityComputer.Builder(); } } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java index 4da718731e9..b1eb71e31f2 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java @@ -22,7 +22,7 @@ import com.carrotsearch.hppc.BitSet; import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -54,7 +54,7 @@ public NodeSimilarity build( configuration, similarityComputer, configuration.concurrency(), - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMetric.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMetric.java index 19024e0d01e..54de85d4455 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMetric.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMetric.java @@ -21,5 +21,6 @@ public enum NodeSimilarityMetric { JACCARD, - OVERLAP + OVERLAP, + COSINE, } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java index ab5ec264d8f..02fb3422315 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityMutateConfig.java @@ -21,14 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface NodeSimilarityMutateConfig extends NodeSimilarityBaseConfig, MutatePropertyConfig, MutateRelationshipConfig { +public interface NodeSimilarityMutateConfig extends NodeSimilarityBaseConfig, + MutateRelationshipPropertyConfig, MutateRelationshipConfig { static NodeSimilarityMutateConfig of(CypherMapWrapper userInput) { NodeSimilarityMutateConfig config = new NodeSimilarityMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java index 0477c38dcff..c12d7b3dcdb 100644 --- a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTreeMutateConfig.java @@ -21,14 +21,15 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface SpanningTreeMutateConfig extends SpanningTreeBaseConfig, MutatePropertyConfig, MutateRelationshipConfig { +public interface SpanningTreeMutateConfig extends SpanningTreeBaseConfig, + MutateRelationshipPropertyConfig, MutateRelationshipConfig { static SpanningTreeMutateConfig of(CypherMapWrapper userInput) { return new SpanningTreeMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java index b0fa97c7ebd..60e0761b4b8 100644 --- a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -51,7 +51,7 @@ public ShortestPathsSteinerAlgorithm build( configuration.delta(), configuration.concurrency(), configuration.applyRerouting(), - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java index 4a08ee51630..b85d996f34f 100644 --- a/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/steiner/SteinerTreeMutateConfig.java @@ -21,14 +21,16 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface SteinerTreeMutateConfig extends SteinerTreeBaseConfig, MutateRelationshipConfig, MutatePropertyConfig { +public interface SteinerTreeMutateConfig extends SteinerTreeBaseConfig, + MutateRelationshipConfig, + MutateRelationshipPropertyConfig { static SteinerTreeMutateConfig of(CypherMapWrapper userInput) { return new SteinerTreeMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/topologicalsort/TopologicalSort.java b/algo/src/main/java/org/neo4j/gds/topologicalsort/TopologicalSort.java index 86f7c36258d..5a9f1f3e07d 100644 --- a/algo/src/main/java/org/neo4j/gds/topologicalsort/TopologicalSort.java +++ b/algo/src/main/java/org/neo4j/gds/topologicalsort/TopologicalSort.java @@ -23,8 +23,8 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicLongArray; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.ParalleLongPageCreator; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -98,7 +98,7 @@ private void initializeInDegrees() { } private void traverse() { - ForkJoinPool forkJoinPool = Pools.createForkJoinPool(concurrency); + ForkJoinPool forkJoinPool = ExecutorServiceUtil.createForkJoinPool(concurrency); var tasks = ConcurrentHashMap.>newKeySet(); ParallelUtil.parallelForEachNode(nodeCount, concurrency, TerminationFlag.RUNNING_TRUE, nodeId -> { diff --git a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java index fa93cf218e2..2a6bd961037 100644 --- a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java +++ b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java @@ -22,7 +22,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.config.SourceNodesConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -150,7 +150,7 @@ private void startWalkers( TOMB, terminationFlag ), - Pools.DEFAULT_SINGLE_THREAD_POOL + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL ).whenComplete((__, ___) -> { progressTracker.endSubTask("RandomWalk"); }); diff --git a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java index 831f7f3d860..39637f4ee02 100644 --- a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalkAlgorithmFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -45,7 +45,7 @@ public RandomWalk build( RandomWalkBaseConfig configuration, ProgressTracker progressTracker ) { - return RandomWalk.create(graph, configuration, progressTracker, Pools.DEFAULT); + return RandomWalk.create(graph, configuration, progressTracker, DefaultPool.INSTANCE); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java b/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java index f76deba421b..4622d95cc86 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/IntersectingTriangleCountFactory.java @@ -23,7 +23,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; import org.neo4j.gds.collections.haa.HugeAtomicLongArray; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -48,7 +48,7 @@ public IntersectingTriangleCount build( return IntersectingTriangleCount.create( graph, configuration, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); } diff --git a/algo/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientMutateConfig.java b/algo/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientMutateConfig.java index 0ac7da99813..541b2637c80 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientMutateConfig.java @@ -21,13 +21,14 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface LocalClusteringCoefficientMutateConfig extends LocalClusteringCoefficientBaseConfig, MutatePropertyConfig { +public interface LocalClusteringCoefficientMutateConfig extends LocalClusteringCoefficientBaseConfig, + MutateNodePropertyConfig { static LocalClusteringCoefficientMutateConfig of(CypherMapWrapper userInput) { return new LocalClusteringCoefficientMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/triangle/TriangleCountMutateConfig.java b/algo/src/main/java/org/neo4j/gds/triangle/TriangleCountMutateConfig.java index b47c567ea74..b0371c48413 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/TriangleCountMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/TriangleCountMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface TriangleCountMutateConfig extends TriangleCountBaseConfig, MutatePropertyConfig { +public interface TriangleCountMutateConfig extends TriangleCountBaseConfig, MutateNodePropertyConfig { static TriangleCountMutateConfig of(CypherMapWrapper userInput) { return new TriangleCountMutateConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java b/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java index 83a9692f4c4..1c07b8b17c5 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/TriangleStreamFactory.java @@ -21,7 +21,7 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; public class TriangleStreamFactory extends GraphAlgorithmFactory { @@ -33,6 +33,6 @@ public String taskName() { @Override public TriangleStream build(Graph graph, TriangleCountBaseConfig configuration, ProgressTracker progressTracker) { - return TriangleStream.create(graph, Pools.DEFAULT, configuration.concurrency()); + return TriangleStream.create(graph, DefaultPool.INSTANCE, configuration.concurrency()); } } diff --git a/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java b/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java index bd83ac7bdac..59b15d4f414 100644 --- a/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java +++ b/algo/src/main/java/org/neo4j/gds/triangle/intersect/GraphIntersect.java @@ -68,8 +68,7 @@ public void intersectAll(long nodeA, IntersectionConsumer consumer) { // iterates over neighbours of A CURSOR neighboursA = cacheA; - // current neighbour of A - long nodeCFromA = NOT_FOUND; + // iterates over neighbours of B CURSOR neighboursB = cacheB; // current neighbour of B @@ -86,7 +85,12 @@ public void intersectAll(long nodeA, IntersectionConsumer consumer) { // check the second node's degree int degreeB = degree(nodeB); if (degreeFilter.test(degreeB)) { - neighboursB = cursorForNode(neighboursB, nodeB, degreeB); + neighboursB = cursorForNode( + neighboursB, + nodeB, + degreeB + ); + // find first neighbour Cb of B with id > B nodeCFromB = neighboursB.skipUntil(nodeB); @@ -95,6 +99,8 @@ public void intersectAll(long nodeA, IntersectionConsumer consumer) { // copy the state of A's cursor neighboursA = copyCursor(neighboursAMain, neighboursA); + var nodeCFromA = NOT_FOUND; //current neighbor from a + if (degreeFilter.test(degree(nodeCFromB))) { // find the first neighbour Ca of A with id >= Cb nodeCFromA = neighboursA.advance(nodeCFromB); diff --git a/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java index e3ace5ed1ac..ce858c0903a 100644 --- a/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/undirected/ToUndirectedAlgorithmFactory.java @@ -22,7 +22,7 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.AdjacencyListBehavior; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -39,7 +39,7 @@ public ToUndirected build( ToUndirectedConfig configuration, ProgressTracker progressTracker ) { - return new ToUndirected(graphStore, configuration, progressTracker, Pools.DEFAULT); + return new ToUndirected(graphStore, configuration, progressTracker, DefaultPool.INSTANCE); } @Override diff --git a/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java index fbb5471a1fd..da8284beb3d 100644 --- a/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/walking/CollapsePathAlgorithmFactory.java @@ -24,7 +24,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import java.util.Collection; @@ -66,7 +66,7 @@ public CollapsePath build( config.allowSelfLoops(), RelationshipType.of(config.mutateRelationshipType()), config.concurrency(), - Pools.DEFAULT + DefaultPool.INSTANCE ); } diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java index 12452cfa332..130ae57ecce 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccAlgorithmFactory.java @@ -21,8 +21,8 @@ import org.neo4j.gds.GraphAlgorithmFactory; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -50,7 +50,7 @@ public Wcc build( } return new Wcc( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, ParallelUtil.DEFAULT_BATCH_SIZE, configuration, progressTracker diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccMutateConfig.java b/algo/src/main/java/org/neo4j/gds/wcc/WccMutateConfig.java index ed7781ffd5e..24be870b966 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccMutateConfig.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccMutateConfig.java @@ -21,13 +21,13 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface WccMutateConfig extends WccBaseConfig, MutatePropertyConfig { +public interface WccMutateConfig extends WccBaseConfig, MutateNodePropertyConfig { static WccMutateConfig of(CypherMapWrapper userInput) { return new WccMutateConfigImpl(userInput); diff --git a/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java b/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java index c12834b6ba3..4a817d7be80 100644 --- a/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/allshortestpaths/MSBFSAllShortestPathsTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.graphbuilder.GraphBuilder; import static org.junit.jupiter.api.Assertions.fail; @@ -76,7 +76,7 @@ void testResults() { .addRelationshipType(RELATIONSHIP) .build() .graph(); - testASP(new MSBFSAllShortestPaths(graph, ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT)); + testASP(new MSBFSAllShortestPaths(graph, ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE)); } private void testASP(final MSBFSASPAlgorithm hugeMSBFSAllShortestPaths) { diff --git a/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java b/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java index 3a58d13e787..53defe79ee9 100644 --- a/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/allshortestpaths/WeightedAllShortestPathsTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.graphbuilder.GraphBuilder; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -88,7 +88,7 @@ void testResults() { final ResultConsumer mock = mock(ResultConsumer.class); - new WeightedAllShortestPaths(graph, Pools.DEFAULT, 4) + new WeightedAllShortestPaths(graph, DefaultPool.INSTANCE, 4) .compute() .forEach(r -> { assertNotEquals(Double.POSITIVE_INFINITY, r.distance); @@ -115,7 +115,7 @@ void shouldThrowIfGraphHasNoRelationshipProperty() { .graph(); UnsupportedOperationException exception = assertThrows(UnsupportedOperationException.class, () -> { - new WeightedAllShortestPaths(graph, Pools.DEFAULT, 4); + new WeightedAllShortestPaths(graph, DefaultPool.INSTANCE, 4); }); assertTrue(exception.getMessage().contains("not supported")); diff --git a/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java b/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java index 397ac21f29e..8a9b33b6b20 100644 --- a/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java +++ b/algo/src/test/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.collections.haa.HugeAtomicDoubleArray; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeByteArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -172,7 +172,7 @@ void computeCorrectResults( var graph = minimize ? minGraph : maxGraph; var approxMaxKCut = new ApproxMaxKCut( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); @@ -214,7 +214,7 @@ void respectMinCommunitySizes(int concurrency) { var approxMaxKCut = new ApproxMaxKCut( maxGraph, - Pools.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java index 161554b0ec3..dee1368f0c6 100644 --- a/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/betweenness/BetweennessCentralityTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -141,7 +141,7 @@ void sampling(int concurrency, TestGraph graph, int samplingSize, Map selectionStrategy.next())).containsExactly( graph.toMappedNodeId("a"), @@ -107,9 +107,9 @@ void neverIncludeZeroDegNodesIfBetterChoicesExist() { TestGraph graph = fromGdl("(), (), (), (), (), (a)-->(), (), (), ()"); SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(graph.toMappedNodeId("a"), selectionStrategy.next()); } @@ -118,18 +118,18 @@ void not100PercentLikelyUnlessMaxDegNode() { TestGraph graph = fromGdl("(a)-->(b), (b)-->(c), (b)-->(d)"); SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1, Optional.of(42L)); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(graph.toMappedNodeId("b"), selectionStrategy.next()); } @Test void selectHighDegreeNode() { SelectionStrategy selectionStrategy = new RandomDegreeSelectionStrategy(1); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); assertEquals(1, samplingSize(selectionStrategy)); - selectionStrategy.init(graph, Pools.DEFAULT, 1); + selectionStrategy.init(graph, DefaultPool.INSTANCE, 1); var isA = selectionStrategy.next(); var isB = selectionStrategy.next(); assertTrue(isA != SelectionStrategy.NONE_SELECTED || isB != SelectionStrategy.NONE_SELECTED); diff --git a/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java index f7a7a33f86c..6b1d8db1526 100644 --- a/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/betweenness/WeightedBetweennessCentralityTest.java @@ -26,7 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -86,7 +86,7 @@ void shouldEqualWithUnweightedWhenWeightsAreEqual() { equallyWeightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.weighted(), - Pools.DEFAULT, + DefaultPool.INSTANCE, 8, ProgressTracker.NULL_TRACKER ); @@ -94,7 +94,7 @@ void shouldEqualWithUnweightedWhenWeightsAreEqual() { equallyWeightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.unweighted(), - Pools.DEFAULT, + DefaultPool.INSTANCE, 8, ProgressTracker.NULL_TRACKER ); @@ -119,7 +119,7 @@ void shouldComputeWithWeights() { weightedGraph, new RandomDegreeSelectionStrategy(7, Optional.of(42L)), ForwardTraverser.Factory.weighted(), - Pools.DEFAULT, + DefaultPool.INSTANCE, 8, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java index 2b24a33c676..df826b3294b 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDirectedTest.java @@ -20,7 +20,7 @@ package org.neo4j.gds.closeness; import org.junit.jupiter.api.Test; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -88,7 +88,7 @@ void testCentrality() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -108,7 +108,7 @@ void testCentralityWithWassermanFaust() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().useWassermanFaust(true).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java index 2a0fb88d9d6..c2256052152 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityDiscoTest.java @@ -22,7 +22,7 @@ import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -67,7 +67,7 @@ void testHuge() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().concurrency(2).useWassermanFaust(true).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java index 46702494f6d..eaa49008d1f 100644 --- a/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/closeness/ClosenessCentralityTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -93,7 +93,7 @@ void testGetCentrality() { var algo = ClosenessCentrality.of( graph, ImmutableClosenessCentralityStreamConfig.builder().build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -116,7 +116,7 @@ void shouldLogProgress() { var algo = ClosenessCentrality.of( graph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java b/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java index 3e8ca895100..a5045d4cd22 100644 --- a/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java +++ b/algo/src/test/java/org/neo4j/gds/conductance/ConductanceTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.TestSupport; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; @@ -134,7 +134,7 @@ void computeCorrectResults( var conductance = new Conductance( orientation == Orientation.NATURAL ? naturalGraph : undirectedGraph, - Pools.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java b/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java index bb38b51b23b..e3f795a479f 100644 --- a/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/degree/DegreeCentralityTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -148,7 +148,7 @@ void shouldComputeCorrectResults(boolean weighted, Orientation orientation, Map< var degreeCentrality = new DegreeCentrality( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); @@ -200,7 +200,7 @@ void testProgressLogging(boolean weighted) { var progressTracker = new TestProgressTracker(progressTask, log, 1, EmptyTaskRegistryFactory.INSTANCE); var degreeCentrality = new DegreeCentrality( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, config, progressTracker ); @@ -225,7 +225,7 @@ void shouldSupportAllOrientations(Orientation orientation) { var degreeCentrality = new DegreeCentrality( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, config, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java index 451e7ff643b..2d549b3b577 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -626,7 +626,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -650,7 +650,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java index 5cfed409328..a1b4fe0032d 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageEmbeddingsGeneratorTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.EnumSource; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.CSRGraphStore; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -74,7 +74,7 @@ void makesEmbeddings(Aggregator.AggregatorType aggregatorType) { var features = GraphSageHelper.initializeSingleLabelFeatures(weightedGraph, config); - var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); GraphSageModelTrainer.ModelTrainResult result = trainModel.train(weightedGraph, features); @@ -84,7 +84,7 @@ void makesEmbeddings(Aggregator.AggregatorType aggregatorType) { config.concurrency(), new SingleLabelFeatureFunction(), config.randomSeed(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -112,7 +112,7 @@ void makesEmbeddingsFromMultiLabelModel(Aggregator.AggregatorType aggregatorType var trainer = new MultiLabelGraphSageTrain( weightedGraph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -125,7 +125,7 @@ void makesEmbeddingsFromMultiLabelModel(Aggregator.AggregatorType aggregatorType config.concurrency(), model.data().featureFunction(), model.trainConfig().randomSeed(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -167,7 +167,7 @@ void embeddingsForNodeFilteredGraph() { var trainer = new SingleLabelGraphSageTrain( filteredGraph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -180,7 +180,7 @@ void embeddingsForNodeFilteredGraph() { config.concurrency(), model.data().featureFunction(), model.trainConfig().randomSeed(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java index 71fb4e5276e..93c3cdc0221 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.beta.generator.PropertyProducer; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageTrainConfigImpl; @@ -120,7 +120,7 @@ void trainsWithRelationshipWeight(AggregatorType aggregatorType) { .modelUser("") .build(); - var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); int nodeCount = 5_000; var bigGraph = RandomGraphGenerator @@ -160,7 +160,7 @@ void trainsWithMeanAggregator(boolean useRelationshipWeight) { .modelName(MODEL_NAME) .build(); - var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var maybeWeights = useRelationshipWeight ? Optional.of("times") : Optional.empty(); @@ -201,7 +201,7 @@ void trainsWithPoolAggregator(boolean useRelationshipWeight) { .modelName(MODEL_NAME) .build(); - var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainModel = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); GraphSageModelTrainer.ModelTrainResult result = trainModel.train(graph, features); Layer[] layers = result.layers(); @@ -251,7 +251,7 @@ void shouldTrainModelWithArrayProperties() { .modelUser("") .build(); - var trainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var result = trainer.train(arrayGraph, arrayFeatures); @@ -276,7 +276,7 @@ void testLosses() { var trainer = new GraphSageModelTrainer( config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -309,7 +309,7 @@ void testLossesWithPoolAggregator() { var trainer = new GraphSageModelTrainer( config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -329,7 +329,7 @@ void testLossesWithPoolAggregator() { void testConvergence() { var trainer = new GraphSageModelTrainer( configBuilder.modelName("convergingModel:)").tolerance(100.0).epochs(10).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -355,13 +355,13 @@ void batchesPerIteration() { var trainResultWithoutSampling = new GraphSageModelTrainer( configBuilder.maybeBatchSamplingRatio(1.0).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); var trainResultWithSampling = new GraphSageModelTrainer( configBuilder.maybeBatchSamplingRatio(0.01).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); @@ -386,7 +386,7 @@ void l2Penalty(double penalty, double expectedLoss) { var result = new GraphSageModelTrainer( config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).train(unweightedGraph, features); @@ -404,8 +404,8 @@ void seededSingleBatch(long seed) { .concurrency(1) .build(); - var trainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); - var otherTrainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); + var otherTrainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var result = trainer.train(unweightedGraph, features); var otherResult = otherTrainer.train(unweightedGraph, features); @@ -424,8 +424,8 @@ void seededMultiBatch(long seed) { .batchSize(5) .build(); - var trainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); - var otherTrainer = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var trainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); + var otherTrainer = new GraphSageModelTrainer(config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var result = trainer.train(unweightedGraph, features); var otherResult = otherTrainer.train(unweightedGraph, features); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java index a201bb9f114..eb6f3cd9b89 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.config.RandomGraphGeneratorConfig; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.CSRGraphStoreUtil; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; @@ -146,7 +146,7 @@ void shouldNotMakeNanEmbeddings(Aggregator.AggregatorType aggregator) { var trainAlgo = new SingleLabelGraphSageTrain( orphanGraph, trainConfig, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -180,7 +180,7 @@ void differentTrainAndPredictionGraph() { .concurrency(1) .build(); - var graphSageTrain = new SingleLabelGraphSageTrain(graph, trainConfig, Pools.DEFAULT, ProgressTracker.NULL_TRACKER, testGdsVersion); + var graphSageTrain = new SingleLabelGraphSageTrain(graph, trainConfig, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion); var model = graphSageTrain.compute(); @@ -205,7 +205,7 @@ void differentTrainAndPredictionGraph() { .batchSize(2) .build(); - var graphSage = new GraphSage(trainGraph, model, streamConfig, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var graphSage = new GraphSage(trainGraph, model, streamConfig, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); assertThat(graphSage.compute().embeddings().size()).isEqualTo(predictNodeCount); } diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java index b4c98ffb212..58aac3b66b2 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/MultiLabelGraphSageTrainTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.embeddings.graphsage.ActivationFunction; import org.neo4j.gds.embeddings.graphsage.Aggregator; @@ -98,7 +98,7 @@ void shouldRunWithDifferentProjectedFeatureSizes(String name, GraphSageTrainConf var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -119,7 +119,7 @@ void shouldStoreMultiLabelFeatureFunctionInModel() { var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -148,7 +148,7 @@ void runsTrainingOnMultiLabelGraph() { var graphSageTrain = new MultiLabelGraphSageTrain( weightedGraph, graphSageTrainConfig, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -191,7 +191,7 @@ void shouldFailUnequalLengthArrays() { var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( unequalGraph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); @@ -216,7 +216,7 @@ void shouldFailMissingArrayProperty(Graph graph, String property, long missingNo var multiLabelGraphSageTrain = new MultiLabelGraphSageTrain( graph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER, testGdsVersion ); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java index a2e414e07b8..e830ae44650 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -422,7 +422,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -446,7 +446,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java index 06712e2221c..8b4ede616d6 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java @@ -44,7 +44,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -280,7 +280,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit .nodes(firstIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var secondMappedToOriginal = HugeLongArray.newArray(nodeCount); @@ -302,7 +302,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit .nodes(secondIdMap) .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var random = new SplittableRandom(42); diff --git a/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java b/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java index d1129e9da9d..99009e5bb62 100644 --- a/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java +++ b/algo/src/test/java/org/neo4j/gds/harmonic/HarmonicCentralityTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.api.Graph; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; @@ -67,7 +67,7 @@ void shouldComputeHarmonicCentrality() { var harmonicCentrality = new HarmonicCentrality( graph, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -89,7 +89,7 @@ void testLogging() { var algo = new HarmonicCentrality( graph, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java b/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java index a5f46a2c531..9e17be857dd 100644 --- a/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java +++ b/algo/src/test/java/org/neo4j/gds/indexInverse/InverseRelationshipsTest.java @@ -27,7 +27,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -79,7 +79,7 @@ void shouldCreateIndexedRelationships(int concurrency) { graphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); assertThat(inverseRelationshipsPerType).hasSize(1); @@ -110,7 +110,7 @@ void shouldIndexMultipleTypes(Object relTypes) { graphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); assertThat(inverseRelationshipsPerType).hasSize(internalTypes.size()); diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java index e406cadefdb..7125941382f 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnConnectedGraphTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; @@ -99,7 +99,7 @@ void testSpreadWithSeed1() { // gain[d|a,b,d,e] : 0 {a already activates d} 1(d) 1(d) = 2/3 =0.667 IdFunction idFunction = variable -> graph.toMappedNodeId(variable); - CELF celf = new CELF(graph, 5, 0.2, 3, Pools.DEFAULT, 2, 0, DEFAULT_BATCH_SIZE, + CELF celf = new CELF(graph, 5, 0.2, 3, DefaultPool.INSTANCE, 2, 0, DEFAULT_BATCH_SIZE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); var celfResult = celf.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java index 6831373860b..43198fb6275 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CELFOnTreeGraphTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -136,7 +136,7 @@ void testSpreadWithSeed1() { 5, 0.51, 3, - Pools.DEFAULT, + DefaultPool.INSTANCE, 1, 10, DEFAULT_BATCH_SIZE, diff --git a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java index 282613cd08b..80e7254bdcc 100644 --- a/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java +++ b/algo/src/test/java/org/neo4j/gds/influenceMaximization/CelfTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import static org.assertj.core.api.Assertions.assertThat; @@ -44,7 +44,7 @@ void shouldNotReturnNegativeGains(int seedSize) { .build() .generate(); - var celf = new CELF(graph, seedSize, 0.1, 3, Pools.DEFAULT, 1, 10, 5, ProgressTracker.NULL_TRACKER).compute(); + var celf = new CELF(graph, seedSize, 0.1, 3, DefaultPool.INSTANCE, 1, 10, 5, ProgressTracker.NULL_TRACKER).compute(); for (var a : celf) { assertThat(a.value).isNotNegative(); } diff --git a/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java b/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java index 252cdff3b0a..121bd5ca2a9 100644 --- a/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java +++ b/algo/src/test/java/org/neo4j/gds/k1coloring/K1ColoringTest.java @@ -32,7 +32,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -69,7 +69,7 @@ void testK1Coloring() { 1000, DEFAULT_BATCH_SIZE, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -110,7 +110,7 @@ void testParallelK1Coloring() { 100, DEFAULT_BATCH_SIZE, 8, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -177,7 +177,7 @@ void everyNodeShouldHaveBeenColored() { 100, DEFAULT_BATCH_SIZE, 8, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -214,7 +214,7 @@ void shouldLogProgress(){ config.maxIterations(), DEFAULT_BATCH_SIZE, config.concurrency(), - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/kcore/KCoreDecompositionTest.java b/algo/src/test/java/org/neo4j/gds/kcore/KCoreDecompositionTest.java index decaeeac217..5f7d54f6c72 100644 --- a/algo/src/test/java/org/neo4j/gds/kcore/KCoreDecompositionTest.java +++ b/algo/src/test/java/org/neo4j/gds/kcore/KCoreDecompositionTest.java @@ -215,4 +215,37 @@ void shouldComputeCoreDecomposition() { } } + + @GdlExtension + @Nested + class K4Graph { + @GdlGraph(orientation = Orientation.UNDIRECTED) + private static final String DB_CYPHER = + "CREATE " + + " (a:node)," + + " (b:node)," + + " (c:node)," + + " (d:node)," + + + "(a)-[:R]->(b)," + + "(a)-[:R]->(c)," + + "(a)-[:R]->(d)," + + "(b)-[:R]->(c)," + + "(b)-[:R]->(d)," + + "(c)-[:R]->(d)"; + + + @Inject + TestGraph graph; + + @Test + void shouldAdvanceScanningDegreeCorrectly() { + var kcore = new KCoreDecomposition(graph, 1, ProgressTracker.NULL_TRACKER, 1).compute(); + assertThat(kcore.degeneracy()).isEqualTo(3); + var coreValues = kcore.coreValues(); + + assertThat(coreValues.toArray()).isEqualTo(new int[]{3, 3, 3, 3}); + + } + } } diff --git a/algo/src/test/java/org/neo4j/gds/kspanningtree/KSpanningTreeTest.java b/algo/src/test/java/org/neo4j/gds/kspanningtree/KSpanningTreeTest.java index d41b8d5874f..301f6c95cc6 100644 --- a/algo/src/test/java/org/neo4j/gds/kspanningtree/KSpanningTreeTest.java +++ b/algo/src/test/java/org/neo4j/gds/kspanningtree/KSpanningTreeTest.java @@ -76,14 +76,6 @@ class KSpanningTreeTest { @Inject private IdFunction idFunction; - private static final int OFFSET = 5; - - @GdlGraph(idOffset = OFFSET, orientation = Orientation.UNDIRECTED, graphNamePrefix = "offset") - private static final String DB_CYPHER_WITH_OFFSET = DB_CYPHER; - - @Inject - private Graph offsetGraph; - private int a, b, c, d, x; @BeforeEach @@ -118,15 +110,6 @@ void testMinimumKSpanningTree() { assertThat(spanningTree.head(b)).isNotEqualTo(spanningTree.head(x)); } - @Test - void testNeoIdsWithOffset() { - var spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, 0, 2, ProgressTracker.NULL_TRACKER).compute(); - var otherSpanningTree = new KSpanningTree(offsetGraph, Prim.MIN_OPERATOR, OFFSET, 2, ProgressTracker.NULL_TRACKER).compute(); - - assertThat(spanningTree.parentArray().toArray()) - .containsExactly(otherSpanningTree.parentArray().toArray()); - } - @Test void shouldProduceSingleConnectedTree() { var factory = GdlFactory.of("CREATE" + diff --git a/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java b/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java index 4c2ead6bfc8..2f60b5a3989 100644 --- a/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java +++ b/algo/src/test/java/org/neo4j/gds/labelpropagation/LabelPropagationTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -83,7 +83,7 @@ void shouldUseOriginalNodeIdWhenSeedPropertyIsMissing() { LabelPropagation lp = new LabelPropagation( graph, ImmutableLabelPropagationStreamConfig.builder().maxIterations(1).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); assertArrayEquals( @@ -109,7 +109,7 @@ void shouldUseSeedProperty() { .seedProperty("seedId") .maxIterations(1) .build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -146,7 +146,7 @@ private void testLPClustering(Graph graph, int batchSize) { LabelPropagation lp = new LabelPropagation( graph, DEFAULT_CONFIG, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); lp.withBatchSize(batchSize); @@ -202,7 +202,7 @@ void shouldLogProgress() { var lp = new LabelPropagation( graph, DEFAULT_CONFIG, - Pools.DEFAULT, + DefaultPool.INSTANCE, testTracker ); diff --git a/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java b/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java index acfea1e94fb..aa25dc610c9 100644 --- a/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java +++ b/algo/src/test/java/org/neo4j/gds/labelpropagation/NonStabilizingLabelPropagationTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -62,7 +62,7 @@ void testLabelPropagationDoesStabilize() { LabelPropagation labelPropagation = new LabelPropagation( graph, ImmutableLabelPropagationStreamConfig.builder().build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); LabelPropagationResult compute = labelPropagation.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java b/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java index 10b2236fcf8..93cd013212f 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -79,7 +79,7 @@ void testGraphAggregation() { Direction.UNDIRECTED, communities, 1L, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, 4, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java b/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java index a82da2cf43b..7724f5827e7 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/GraphWithSelfLoopTest.java @@ -23,7 +23,8 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -70,7 +71,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); @@ -86,7 +87,7 @@ void shouldAggregateCorrectly(){ Direction.UNDIRECTED, communities, 2L, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, 4, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -121,7 +122,7 @@ void shouldMaintainPartition() { Direction.UNDIRECTED, refinedCommunities, 2, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -164,7 +165,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 2, - Pools.DEFAULT, + DefaultPool.INSTANCE, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -180,7 +181,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java b/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java index 2685788beb0..7f1d0c56d85 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java @@ -30,7 +30,7 @@ import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.config.RandomGraphGeneratorConfig; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -219,7 +219,7 @@ void shouldMaintainPartition() { Direction.UNDIRECTED, refinedCommunities, 4, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java b/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java index 9264951ada4..5d11a2e780b 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/ModularityComputerTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -83,7 +83,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); @@ -102,7 +102,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -116,7 +116,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / graph.relationshipCount(), 1.0 / graph.relationshipCount(), 4, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java index 89a8ebc17d3..cab2f6b6dcd 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseKarateTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -73,7 +73,7 @@ void testRefinementPhase() { 0.01, 19L, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java index 2321b3638c9..f9597a50a14 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/RefinementPhaseTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -80,7 +80,7 @@ void shouldRefine() { 0.01, 19L, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); var refinementResult = refinement.run(); diff --git a/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java b/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java index 65f3fbd70e4..92de2482424 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/WeightedModularityComputerTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -84,7 +84,7 @@ void shouldCalculateModularityCorrectly() { 1.0 / (4.0 * graph.relationshipCount()), 1.0 / (4.0 * graph.relationshipCount()), 4, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); @@ -103,7 +103,7 @@ void shouldCalculateModularityInSummaryGraph() { Direction.UNDIRECTED, localCommunities, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, 1, TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER @@ -117,7 +117,7 @@ void shouldCalculateModularityInSummaryGraph() { 1.0 / (4 * graph.relationshipCount()), 1.0 / (4 * graph.relationshipCount()), 4, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.EmptyProgressTracker.NULL_TRACKER ); assertThat(modularity).isCloseTo(0.4230, Offset.offset(1e-3)); diff --git a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java index 65f52d8f258..db34cff670a 100644 --- a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java +++ b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -155,7 +155,7 @@ void testUnweighted() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -204,7 +204,7 @@ void testWeighted() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -253,7 +253,7 @@ void testSeeded() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -304,7 +304,7 @@ void testTolerance() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -340,7 +340,7 @@ void testMaxLevels() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -457,7 +457,7 @@ void testCanBeInterruptedByTxCancellation() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); louvain.setTerminationFlag(terminationFlag); @@ -494,7 +494,7 @@ void testLogging() { config.tolerance(), config.concurrency(), progressTracker, - Pools.DEFAULT + DefaultPool.INSTANCE ); @@ -522,7 +522,7 @@ void shouldThrowOnNegativeSeed() { config.tolerance(), config.concurrency(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); algorithm.setTerminationFlag(TerminationFlag.RUNNING_TRUE); @@ -554,7 +554,7 @@ void shouldGiveSameResultWithCalculator() { TOLERANCE_DEFAULT, 4, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var result = louvain.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java index dfb8f26d47c..458b1b3bd09 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/FootballTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.compat.Neo4jProxy; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -127,7 +127,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java index 482a520b223..96f2f5c0043 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryTree; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.extension.GdlExtension; @@ -243,7 +243,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java index 312d3ce0d26..db0ef84da0f 100644 --- a/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationWithoutOrientationTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryTree; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.extension.GdlExtension; @@ -252,7 +252,7 @@ private ModularityOptimizationResult compute( properties, concurrency, minBatchSize, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).compute(); } diff --git a/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java b/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java index 88a176bbf84..f0b4fd9a4a6 100644 --- a/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java +++ b/algo/src/test/java/org/neo4j/gds/msbfs/MultiSourceBFSAccessMethodsTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.api.RelationshipIterator; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; @@ -85,7 +85,7 @@ void testWithPredecessor() { new long[]{0, 1} ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); verify(bfsConsumerMock).accept(1, 0, toList(1)); verify(bfsConsumerMock).accept(2, 0, toList(2)); @@ -126,7 +126,7 @@ void testWithANP() { new long[]{0, 1} ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); verify(mock).accept(3, 1, toList(1, 2)); verify(mock).accept(4, 1, toList(1, 2)); @@ -148,7 +148,7 @@ void testPredecessorWithAllSources() { (i, p, d, s) -> mock.accept(i + 1, p + 1, d, toList(s, x -> x + 1)) ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); verify(mock).accept(1, 3, 1, toList(3)); verify(mock).accept(1, 4, 1, toList(4)); @@ -199,7 +199,7 @@ void testANPWithAllSources() { (i, d, s) -> mock.accept(i + 1, d, toList(s, x -> x + 1)) ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); verify(mock).accept(1, 1, toList(3, 4)); verify(mock).accept(2, 1, toList(3, 4)); @@ -270,7 +270,7 @@ void testParallel() { } } ); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); }); for (int i = 0; i < maxNodes; i++) { @@ -382,7 +382,7 @@ public Stream streamRelationships(long nodeId, double fallba } }, sources); - msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, Pools.DEFAULT); + msbfs.run(ConcurrencyConfig.DEFAULT_CONCURRENCY, DefaultPool.INSTANCE); for (int i = 0; i < seen.length; i++) { final int[] nodeSeen = seen[i]; diff --git a/algo/src/test/java/org/neo4j/gds/pagerank/PageRankConfigTest.java b/algo/src/test/java/org/neo4j/gds/pagerank/PageRankConfigTest.java new file mode 100644 index 00000000000..4aa6ff1aa7c --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/pagerank/PageRankConfigTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pagerank; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@GdlExtension +public class PageRankConfigTest { + + @GdlGraph + private static final String DB_CYPHER = + "CREATE " + + " (a:node)," + + " (b:node)," + + "(a)-[:R]->(b)"; + + + @Inject + GraphStore graphStore; + + @Test + void shouldNotAllowNegativeSourceNodes() { + + var config = PageRankStreamConfigImpl.builder() + .sourceNodes(List.of(-1337)).build(); + + assertThatThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + config.internalRelationshipTypes(graphStore) + )) + .hasMessageContaining("Negative node ids are not supported for the field `sourceNodes`"); + + } + + @Test + void shouldNotAllowNonExistantSourceNodes() { + + var config = PageRankStreamConfigImpl.builder() + .sourceNodes(List.of(421337)).build(); + + assertThatThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + config.internalRelationshipTypes(graphStore) + )) + .hasMessageContaining("sourceNodes nodes do not exist in the in-memory graph: ['421337']"); + + } +} diff --git a/algo/src/test/java/org/neo4j/gds/pagerank/PageRankTest.java b/algo/src/test/java/org/neo4j/gds/pagerank/PageRankTest.java index 5db2c5025b7..e7b63ab09d0 100644 --- a/algo/src/test/java/org/neo4j/gds/pagerank/PageRankTest.java +++ b/algo/src/test/java/org/neo4j/gds/pagerank/PageRankTest.java @@ -169,15 +169,19 @@ void withSourceNodes(String sourceNodesString, String expectedPropertyKey) { } } - @Test - void shouldLogProgress() { + @ParameterizedTest + @EnumSource(Mode.class) + void shouldLogProgress(Mode mode) { var maxIterations = 10; var config = ImmutablePageRankConfig.builder() .maxIterations(maxIterations) .build(); - var progressTask = PageRankAlgorithmFactory.pagerankProgressTask(graph, config); + var factory = new PageRankAlgorithmFactory<>(mode); + + var progressTask = factory.progressTask(graph, config); var log = Neo4jProxy.testLog(); + var progressTracker = new TestProgressTracker( progressTask, log, @@ -185,7 +189,12 @@ void shouldLogProgress() { EmptyTaskRegistryFactory.INSTANCE ); - runOnPregel(graph, config, Mode.PAGE_RANK, progressTracker); + factory.build( + graph, + config, + progressTracker + ) + .compute(); var progresses = progressTracker.getProgresses().stream() .filter(it -> it.get() > 0) @@ -207,22 +216,26 @@ void shouldLogProgress() { .extracting(removingThreadId()) .contains( formatWithLocale( - "PageRank :: Compute iteration %d of %d :: Start", + "%s :: Compute iteration %d of %d :: Start", + mode.taskName(), iteration, config.maxIterations() ), formatWithLocale( - "PageRank :: Compute iteration %d of %d :: Finished", + "%s :: Compute iteration %d of %d :: Finished", + mode.taskName(), iteration, config.maxIterations() ), formatWithLocale( - "PageRank :: Master compute iteration %d of %d :: Start", + "%s :: Master compute iteration %d of %d :: Start", + mode.taskName(), iteration, config.maxIterations() ), formatWithLocale( - "PageRank :: Master compute iteration %d of %d :: Finished", + "%s :: Master compute iteration %d of %d :: Finished", + mode.taskName(), iteration, config.maxIterations() ) @@ -231,8 +244,8 @@ void shouldLogProgress() { assertThat(messages) .extracting(removingThreadId()) .contains( - "PageRank :: Start", - "PageRank :: Finished" + formatWithLocale("%s :: Start", mode.taskName()), + formatWithLocale("%s :: Finished", mode.taskName()) ); } @@ -681,16 +694,13 @@ PageRankResult runOnPregel(Graph graph, PageRankConfig config) { } PageRankResult runOnPregel(Graph graph, PageRankConfig config, Mode mode) { - return runOnPregel(graph, config, mode, ProgressTracker.NULL_TRACKER); - } - - PageRankResult runOnPregel(Graph graph, PageRankConfig config, Mode mode, ProgressTracker progressTracker) { return new PageRankAlgorithmFactory<>(mode) .build( graph, config, - progressTracker + ProgressTracker.NULL_TRACKER ) .compute(); } + } diff --git a/algo/src/test/java/org/neo4j/gds/paths/ShortestPathConfigTest.java b/algo/src/test/java/org/neo4j/gds/paths/ShortestPathConfigTest.java index 2354bcef27f..7700e926052 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/ShortestPathConfigTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/ShortestPathConfigTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.paths.dijkstra.config.ShortestPathDijkstraStreamConfigImpl; +import org.neo4j.gds.paths.yens.config.ShortestPathYensStreamConfigImpl; import org.neo4j.kernel.impl.core.NodeEntity; import static org.assertj.core.api.Assertions.assertThat; @@ -55,6 +56,33 @@ void shouldAllowNodeIds() { assertThat(config.targetNode()).isEqualTo(1337L); } + @Test + void shouldNotAllowNegativeSourceNode() { + + var config = ShortestPathYensStreamConfigImpl.builder() + .k(1) + .sourceNode(-1337) + .targetNode(0); + + assertThatThrownBy(() -> config.build()) + .hasMessageContaining("Negative node ids are not supported for the field `sourceNode`"); + + } + + @Test + void shouldNotAllowNegativeTargetNode() { + + var config = ShortestPathYensStreamConfigImpl.builder() + .k(1) + .sourceNode(0) + .targetNode(-1337); + + assertThatThrownBy(() -> config.build()) + .hasMessageContaining("Negative node ids are not supported for the field `targetNode`"); + + } + + @Test void shouldThrowErrorOnUnsupportedType() { var cypherMapWrapper = CypherMapWrapper diff --git a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java index e20bd89e0d1..bf6470e3644 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java @@ -36,7 +36,7 @@ import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -142,7 +142,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -172,7 +172,7 @@ void singleSourceFromDisconnectedNode(double delta, int concurrency, long idOffs .build(); var paths = DeltaStepping - .of(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -191,7 +191,7 @@ void shouldLogProgress() { var testLog = Neo4jProxy.testLog(); var progressTracker = new TestProgressTracker(progressTask, testLog, 1, EmptyTaskRegistryFactory.INSTANCE); - DeltaStepping.of(graph, config, Pools.DEFAULT, progressTracker) + DeltaStepping.of(graph, config, DefaultPool.INSTANCE, progressTracker) .compute() .pathSet(); @@ -279,7 +279,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -335,7 +335,7 @@ void singleSource(double delta, int concurrency, long idOffset) { .build(); var paths = DeltaStepping - .of(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER) + .of(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER) .compute() .pathSet(); @@ -369,7 +369,7 @@ void shouldGiveSameResultsAsDijkstra() { var deltaStepping = DeltaStepping.of( newGraph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/paths/traverse/BfsConfigTest.java b/algo/src/test/java/org/neo4j/gds/paths/traverse/BfsConfigTest.java new file mode 100644 index 00000000000..5dfa1103552 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/paths/traverse/BfsConfigTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.paths.traverse; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.IdFunction; +import org.neo4j.gds.extension.Inject; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@GdlExtension +public class BfsConfigTest { + + @GdlGraph + private static final String DB_CYPHER = + "CREATE " + + " (a:node)," + + " (b:node)," + + "(a)-[:R]->(b)"; + + + @Inject + GraphStore graphStore; + + @Inject + IdFunction idFunction; + + @Test + void shouldNotAllowNegativeTargetNodes() { + var config = BfsStreamConfigImpl.builder() + .sourceNode(0L) + .targetNodes(List.of(idFunction.of("a"), -1337)).build(); + + assertThatThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + config.internalRelationshipTypes(graphStore) + )) + .hasMessageContaining("Negative node ids are not supported for the field `targetNodes`"); + + } + + @Test + void failOnInvalidEndNodes() { + + var config = BfsStreamConfigImpl.builder() + .sourceNode(idFunction.of("a")) + .targetNodes(List.of(idFunction.of("b"), 421337)).build(); + + assertThatThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + config.internalRelationshipTypes(graphStore) + )) + .hasMessageContaining("targetNodes nodes do not exist in the in-memory graph: ['421337']"); + } +} diff --git a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java index 0ff42febad6..222f31e5be6 100644 --- a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java +++ b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesMissingPropsTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -55,7 +55,7 @@ void partialArrays() { .nodeProperties(List.of("arrayOn4", "arrayOn1")) .scaler(Max.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -73,7 +73,7 @@ void testMissingScalar() { .nodeProperties(List.of("a", "b", "c")) .scaler(StdScore.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); diff --git a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java index 057477e127b..026fa49575f 100644 --- a/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java +++ b/algo/src/test/java/org/neo4j/gds/scaleproperties/ScalePropertiesTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -79,7 +79,7 @@ void scaleSingleProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .concurrency(1) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -98,7 +98,7 @@ void scaleMultipleProperties() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .concurrency(1) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var result = algo.compute(); var resultProperties = result.scaledProperties().toArray(); @@ -130,14 +130,14 @@ void parallelScale() { bigGraph, config.concurrency(4).build(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute().scaledProperties(); var expected = new ScaleProperties( bigGraph, config.concurrency(1).build(), ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute().scaledProperties(); IntStream.range(0, nodeCount).forEach(id -> assertEquals(expected.get(id)[0], parallelResult.get(id)[0])); @@ -150,7 +150,7 @@ void scaleArrayProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var actual = new ScaleProperties(graph, arrayConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var actual = new ScaleProperties(graph, arrayConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); @@ -159,7 +159,7 @@ void scaleArrayProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var expected = new ScaleProperties(graph, singlePropConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var expected = new ScaleProperties(graph, singlePropConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); @@ -175,13 +175,13 @@ void supportLongAndFloatArrays(String scaler) { var longArrayBConfig = baseConfigBuilder.nodeProperties(List.of("longArrayB")).build(); var doubleArrayBConfig = baseConfigBuilder.nodeProperties(List.of("floatArrayB")).build(); - var expected = new ScaleProperties(graph, bConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var expected = new ScaleProperties(graph, bConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); - var actualLong = new ScaleProperties(graph, longArrayBConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var actualLong = new ScaleProperties(graph, longArrayBConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); - var actualDouble = new ScaleProperties(graph, doubleArrayBConfig, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var actualDouble = new ScaleProperties(graph, doubleArrayBConfig, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); @@ -197,7 +197,7 @@ void supportDoubleArrays() { var config = baseConfigBuilder.nodeProperties(List.of("doubleArray")).build(); var expected = new double[][]{new double[]{0.0}, new double[]{0.2499999722444236}, new double[]{.5}, new double[]{0.7500000277555764}, new double[]{1.0}}; - var actual = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT) + var actual = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE) .compute() .scaledProperties(); @@ -211,7 +211,7 @@ void failOnArrayPropertyWithUnequalLength() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var error = assertThrows(IllegalArgumentException.class, algo::compute); assertThat(error.getMessage(), containsString( @@ -226,7 +226,7 @@ void failOnNonExistentProperty() { .scaler(MinMax.buildFrom(CypherMapWrapper.empty())) .build(); - var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, Pools.DEFAULT); + var algo = new ScaleProperties(graph, config, ProgressTracker.NULL_TRACKER, DefaultPool.INSTANCE); var error = assertThrows(IllegalArgumentException.class, algo::compute); assertThat(error.getMessage(), containsString("Node property `IMAGINARY_PROP` not found in graph")); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/EuclideanTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/EuclideanTest.java new file mode 100644 index 00000000000..6769db1727b --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/EuclideanTest.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.knn.metrics; + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class EuclideanTest { + + @Test + void doubleArrays() { + var left = new double[] {0.008706967509313894d, 0.0d, 0.004919643078839634d, 0.007029592654453866d, 0.0d, 0.0d, 0.1257682851778286d, 0.016530303738513153d, 0.012745441170181648d, 0.0d}; + var right = new double[] {0.008706967509313894d, 0.0d, 0.0041533258820686736d, 0.003698167604532474d, 0.0d, 0.0d, 0.14438303839785882d, 0.0017742178454893931d, 0.008013536500447603d, 0.0d}; + + var metric = Euclidean.doubleMetric(left, right); + + assertThat(metric).isCloseTo(0.976123304363789d, Offset.offset(1e-5)); + } + + @Test + void floatArrays() { + var left = new float[] {0.008706967509313894f, 0.0f, 0.004919643078839634f, 0.007029592654453866f, 0.0f, 0.0f, 0.1257682851778286f, 0.016530303738513153f, 0.012745441170181648f, 0.0f}; + var right = new float[] {0.008706967509313894f, 0.0f, 0.0041533258820686736f, 0.003698167604532474f, 0.0f, 0.0f, 0.14438303839785882f, 0.0017742178454893931f, 0.008013536500447603f, 0.0f}; + + var metric = Euclidean.floatMetric(left, right); + + assertThat(metric).isCloseTo(0.976123304363789d, Offset.offset(1e-5)); + } +} diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputerTest.java new file mode 100644 index 00000000000..9da7449cd8f --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/CosineSimilarityComputerTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.nodesim; + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class CosineSimilarityComputerTest { + + @Test + void shouldComputeUnweightedCosineSimilarity() { + var array1 = new long[]{0, 1, 2, 4, 6}; + var array2 = new long[]{0, 1, 2, 3, 5, 6, 7}; + var similarityComputer = new CosineSimilarityComputer(0); + + var cosineSimilarity = similarityComputer.computeSimilarity(array1, array2); + + assertThat(cosineSimilarity).isCloseTo(0.6761234037828131, Offset.offset(1e-5)); + } + + @Test + void shouldComputeUnweightedCosineSimilarityRespectingCutoff() { + var array1 = new long[]{0, 1, 2, 4, 6}; + var array2 = new long[]{0, 1, 2, 3, 5, 6, 7}; + var similarityComputer = new CosineSimilarityComputer(0.7); + + var cosineSimilarity = similarityComputer.computeSimilarity(array1, array2); + + assertThat(cosineSimilarity).isNaN(); + } + + @Test + void shouldComputeCosineOnArraysWithNoMissingElements() { + var similarityComputer = new CosineSimilarityComputer(0); + var array1 = new long[]{0, 1, 2}; + var weight1 = new double[]{0.5, 0.2, 0.6}; + var array2 = new long[]{0, 1, 2}; + var weight2 = new double[]{1.3, 0.9, 0.2}; + + assertThat(similarityComputer.computeWeightedSimilarity(array1, array2, weight1, weight2)).isCloseTo( + 0.73935, + Offset.offset(1e-5) + ); + } + + @Test + void shouldComputeCosineOnArraysWithNoMissingElementsRespectingCutoff() { + var similarityComputer = new CosineSimilarityComputer(1); + var array1 = new long[]{0, 1, 2}; + var weight1 = new double[]{0.5, 0.2, 0.6}; + var array2 = new long[]{0, 1, 2}; + var weight2 = new double[]{1.3, 0.9, 0.2}; + + assertThat(similarityComputer.computeWeightedSimilarity(array1, array2, weight1, weight2)).isNaN(); + } + + @Test + void shouldComputeCosineOnArraysWithMissingElements() { + var similarityComputer = new CosineSimilarityComputer(0); + var array1 = new long[]{0, 1, 2, 4, 6}; + var weight1 = new double[]{0.5, 0.2, 0.6, 0.8, 1}; + var array2 = new long[]{0, 1, 2, 3, 5, 6, 7}; + var weight2 = new double[]{1.3, 0.9, 0.2, 4.2, 1, 2, 3}; + + assertThat(similarityComputer.computeWeightedSimilarity(array1, array2, weight1, weight2)).isCloseTo( + 0.33344, + Offset.offset(1e-5) + ); + } + +} diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputerTest.java index f36a1ba506b..f944ddc92a5 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/MetricSimilarityComputerTest.java @@ -21,6 +21,7 @@ import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; class MetricSimilarityComputerTest { @@ -31,4 +32,11 @@ void shouldThrowOnWrongMetric2() { .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("ovErLaPPPPP is not a valid metric"); } + + @Test + void shouldParseCosineSimilarityComputer() { + var cosineComputer = MetricSimilarityComputer.parse("CosiNe").build(0); + assertThat(cosineComputer) + .isExactlyInstanceOf(CosineSimilarityComputer.class); + } } diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java index d3ae7902e0c..599fc2d1348 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java @@ -23,7 +23,7 @@ import org.neo4j.gds.BaseTest; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import static org.neo4j.gds.graphbuilder.TransactionTerminationTestUtils.assertTerminates; @@ -44,7 +44,7 @@ void shouldTerminate() { var nodeSimilarity = NodeSimilarity.create( graph, NodeSimilarityTest.configBuilder().concurrency(1).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); nodeSimilarity.setTerminationFlag(terminationFlag); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index 17d10622ca4..4e8a2448643 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -32,7 +32,7 @@ import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.mem.MemoryTree; @@ -291,7 +291,7 @@ void shouldComputeWeightedForSupportedDirections(Orientation orientation, int co NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().relationshipWeightProperty("prop").concurrency(concurrency).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -311,7 +311,7 @@ void shouldComputeForSupportedDirections(Orientation orientation, int concurrenc NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -331,7 +331,7 @@ void shouldComputeTopNForSupportedDirections(Orientation orientation, int concur NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).topN(1).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -351,7 +351,7 @@ void shouldComputeNegativeTopNForSupportedDirections(Orientation orientation, in NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).bottomN(1).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -373,7 +373,7 @@ void shouldComputeTopKForSupportedDirections(Orientation orientation, int concur NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().topK(1).concurrency(concurrency).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -397,7 +397,7 @@ void shouldComputeNegativeTopKForSupportedDirections(Orientation orientation, in .topK(10) .bottomK(1) .build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -427,7 +427,7 @@ void shouldComputeWithSimilarityCutoffForSupportedDirections(Orientation orienta NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).similarityCutoff(0.1).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -450,7 +450,7 @@ void shouldComputeWithDegreeCutoffForSupportedDirections(Orientation orientation NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().degreeCutoff(2).concurrency(concurrency).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -471,7 +471,7 @@ void shouldComputeForUndirectedGraphs(int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( undirectedGraph, configBuilder().concurrency(concurrency).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); Set result = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -483,7 +483,7 @@ void shouldComputeForUnionGraphs() { NodeSimilarity nodeSimilarity = NodeSimilarity.create( naturalGraph, configBuilder().concurrency(1).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); var result1 = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -491,7 +491,7 @@ void shouldComputeForUnionGraphs() { nodeSimilarity = NodeSimilarity.create( naturalUnionGraph, configBuilder().concurrency(1).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); var result2 = nodeSimilarity.computeToStream().collect(Collectors.toSet()); @@ -507,7 +507,7 @@ void shouldComputeSimilarityGraphInAllSupportedDirections(Orientation orientatio NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -558,7 +558,7 @@ void shouldComputeToGraphWithUnusedNodesInInputGraph(Orientation orientation, in .topK(100) .topN(1) .build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -596,7 +596,7 @@ void shouldIgnoreLoops(Orientation orientation, int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).topN(1).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -623,7 +623,7 @@ void shouldIgnoreParallelEdges(Orientation orientation, int concurrency) { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(concurrency).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -840,7 +840,7 @@ void shouldLogProgress(int concurrency) { NodeSimilarity.create( graph, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).compute().streamResult().count(); @@ -880,7 +880,7 @@ void shouldGiveCorrectResultsWithOverlap() { NodeSimilarity nodeSimilarity = NodeSimilarity.create( graph, configBuilder().concurrency(1).similarityMetric(MetricSimilarityComputer.parse("ovErLaP")).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -898,7 +898,7 @@ void shouldGiveCorrectResultsWithOverlap() { .concurrency(1) .similarityMetric(MetricSimilarityComputer.parse("ovErLaP")) .build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -934,7 +934,7 @@ void shouldWorkForAllDegreeBoundsCombinations(int lowBound, int upperBound, Stri NodeSimilarity nodeSimilarity = NodeSimilarity.create( naturalGraph, configBuilder().upperDegreeCutoff(upperBound).degreeCutoff(lowBound).build(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java index d71a5bb42ba..858524a05f0 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/SimilarityGraphBuilderTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.huge.UnionGraph; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -73,7 +73,7 @@ void testConstructionFromHugeGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( unlabelledGraph, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, TerminationFlag.RUNNING_TRUE ); @@ -96,7 +96,7 @@ void testConstructionFromUnionGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( graph, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, TerminationFlag.RUNNING_TRUE ); @@ -131,7 +131,7 @@ void testConstructFromFilteredGraph() { SimilarityGraphBuilder similarityGraphBuilder = new SimilarityGraphBuilder( filteredIdMap, 1, - Pools.DEFAULT, + DefaultPool.INSTANCE, TerminationFlag.RUNNING_TRUE ); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java index 0c4ab250cad..b7c371db0d7 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java @@ -22,7 +22,7 @@ import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -58,7 +58,7 @@ void shouldWorkWithUnionGraph(){ .topN(1) .build(); - var nodeSimilarity = NodeSimilarity.create(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var nodeSimilarity = NodeSimilarity.create(graph, config, DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER); var result = nodeSimilarity.compute().streamResult().findFirst().get(); //input should be (0 + 10 + 0)/ (5 + 10 + 8) = 10/23 diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java index 5d21acddb47..b826bccc72a 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathSteinerAlgorithmExtendedTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -155,7 +155,7 @@ void shouldWorkCorrectly(double delta, int binSizeThreshold) { false, binSizeThreshold, //setting custom threshold for such a small graph allows to not examine everything in a single iteration - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); @@ -175,7 +175,7 @@ void shouldWorkCorrectlyWithLineGraph() { 2.0, 1, false, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ) .compute(); @@ -203,7 +203,7 @@ void deltaSteppingShouldWorkCorrectly() { isTerminal, 1, SteinerBasedDeltaStepping.BIN_SIZE_THRESHOLD, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); var result = deltaSteiner.compute().pathSet(); @@ -229,7 +229,7 @@ void shouldWorkIfRevisitsVertices() { 2.0, 1, false, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); @@ -258,7 +258,7 @@ void shouldWorkOnTriangle() { 2.0, 1, false, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java index 4e97db509ab..688070202ad 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmReroutingTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -189,7 +189,7 @@ void shouldPruneUnusedIfRerouting() { 2.0, 1, false, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResult.totalCost()).isEqualTo(7.0); @@ -203,7 +203,7 @@ void shouldPruneUnusedIfRerouting() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(4.0); @@ -223,7 +223,7 @@ void shouldPruneUnusedIfReroutingOnInvertedIndex() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(4.0); @@ -242,7 +242,7 @@ void rerouteShouldNotCreateLoops() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); var parent = steinerResult.parentArray().toArray(); @@ -267,7 +267,7 @@ void rerouteShouldNotCreateLoopsOnInvertedIndex() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); var parent = steinerResult.parentArray().toArray(); @@ -294,7 +294,7 @@ void shouldWorkForUnreachableAndReachableTerminals() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerTreeResult.effectiveTargetNodesCount()).isEqualTo(2); @@ -315,7 +315,7 @@ void shouldWorkIfNoReachableTerminals() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerTreeResult.effectiveTargetNodesCount()).isEqualTo(0); @@ -473,7 +473,7 @@ void shouldNotGetOptimalWithoutBetterRerouting() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(25.0); @@ -498,7 +498,7 @@ void shouldHandleMultiplePruningsOnSameTreeAndGetBetter() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(22.0); @@ -522,7 +522,7 @@ void shouldNotPruneUnprunableNodes() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(170.0 - 19); @@ -546,7 +546,7 @@ void shouldTakeAdvantageOfNewSingleParents() { 2.0, 1, true, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); assertThat(steinerResultWithReroute.totalCost()).isEqualTo(20); @@ -554,5 +554,3 @@ void shouldTakeAdvantageOfNewSingleParents() { } } - - diff --git a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java index 9abd7b58c0c..d057bcb99e7 100644 --- a/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java +++ b/algo/src/test/java/org/neo4j/gds/steiner/ShortestPathsSteinerAlgorithmTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -75,7 +75,7 @@ void shouldWorkCorrectly() { 2.0, 1, false, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).compute(); var pruned = ShortestPathsSteinerAlgorithm.PRUNED; diff --git a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java index f1dde921af3..689a1b8186a 100644 --- a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java +++ b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java @@ -34,7 +34,7 @@ import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -95,7 +95,7 @@ void testWithDefaultConfig() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); List result = randomWalk.compute().collect(Collectors.toList()); @@ -136,7 +136,7 @@ private List runRandomWalkSeeded(Node2VecStreamConfig config, Graph grap graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); return randomWalk.compute().collect(Collectors.toList()); @@ -149,7 +149,7 @@ void testSampleFromMultipleRelationshipTypes() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); int expectedNumberOfWalks = config.walksPerNode() * 3; @@ -190,7 +190,7 @@ void returnFactorShouldMakeWalksIncludeStartNodeMoreOften() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var nodeCounter = new HashMap(); @@ -250,7 +250,7 @@ void largeInOutFactorShouldMakeTheWalkKeepTheSameDistance() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var nodeCounter = new HashMap(); @@ -299,7 +299,7 @@ void shouldRespectRelationshipWeights() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var nodeCounter = new HashMap(); @@ -336,7 +336,7 @@ void failOnInvalidRelationshipWeights(double invalidWeight) { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ) ).isInstanceOf(RuntimeException.class) .hasMessage( @@ -371,7 +371,7 @@ void parallelWeighted() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(randomWalk.compute().collect(Collectors.toList())) @@ -400,7 +400,7 @@ void testWithConfiguredOffsetStartNodes() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); assertThat(randomWalk.compute().collect(Collectors.toList())) @@ -423,7 +423,7 @@ void testSetTerminationFlagAndMultipleRuns() { graph, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ); var stream = randomWalk.compute(); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java index 98501e94fbb..7a146f23688 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountFilteredGraphTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.extension.Neo4jGraph; import java.util.Collections; @@ -40,7 +40,7 @@ class IntersectingTriangleCountFilteredGraphTest extends BaseTest { - @Neo4jGraph + @Neo4jGraph(offsetIds = true) static final String DB_CYPHER = "CREATE " + " (a1:A)" + ", (a2:A)" + @@ -78,7 +78,7 @@ void testUnionGraphWithNodeFilter() { Optional.empty() ); var config = ImmutableTriangleCountBaseConfig.builder().build(); - var triangleCount = IntersectingTriangleCount.create(graph, config, Pools.DEFAULT); + var triangleCount = IntersectingTriangleCount.create(graph, config, DefaultPool.INSTANCE); var triangleCountResult = triangleCount.compute(); assertThat(triangleCountResult.globalTriangles()).isEqualTo(1); var triangles = triangleCountResult.localTriangles(); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java index 5d189682de5..a3403f05824 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/IntersectingTriangleCountTest.java @@ -27,7 +27,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import java.util.stream.Stream; @@ -531,7 +531,7 @@ private TriangleCountResult compute(Graph graph) { } private TriangleCountResult compute(Graph graph, TriangleCountBaseConfig config) { - return IntersectingTriangleCount.create(graph, config, Pools.DEFAULT).compute(); + return IntersectingTriangleCount.create(graph, config, DefaultPool.INSTANCE).compute(); } private static Graph fromGdl(String gdl) { diff --git a/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java b/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java index 3e7722bb1c3..23f5eea35f7 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/LargeIntersectingTriangleCountTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.PagedAtomicIntegerArray; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; @@ -71,7 +71,7 @@ void testQueue() { var result = IntersectingTriangleCount.create( graph, defaultConfigBuilder().build(), - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); assertEquals(TRIANGLE_COUNT, result.globalTriangles()); assertTriangles(result.globalTriangles()); @@ -83,7 +83,7 @@ void testQueueParallel() { var result = IntersectingTriangleCount.create( graph, defaultConfigBuilder().concurrency(4).build(), - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); assertEquals(TRIANGLE_COUNT, result.globalTriangles()); assertTriangles(result.globalTriangles()); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/TriangleCountMaxDegreeTest.java b/algo/src/test/java/org/neo4j/gds/triangle/TriangleCountMaxDegreeTest.java new file mode 100644 index 00000000000..e311c8a9ab1 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/triangle/TriangleCountMaxDegreeTest.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.triangle; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.beta.generator.RandomGraphGenerator; +import org.neo4j.gds.beta.generator.RelationshipDistribution; +import org.neo4j.gds.core.concurrency.DefaultPool; + +import static org.assertj.core.api.Assertions.assertThat; + +class TriangleCountMaxDegreeTest { + + @Test + void shouldWorkWithMaxDegree(){ + var graph= RandomGraphGenerator.builder().nodeCount(24).averageDegree(23).seed(101) + .relationshipDistribution( + RelationshipDistribution.RANDOM) + .direction(Direction.UNDIRECTED) + .build().generate(); + + TriangleCountBaseConfig config = ImmutableTriangleCountBaseConfig + .builder() + .maxDegree(100) + .build(); + + var tc = IntersectingTriangleCount.create(graph, config, DefaultPool.INSTANCE).compute(); + assertThat(tc.globalTriangles()).isEqualTo(1262L); + + } + +} diff --git a/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java b/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java index 665b2b15981..6a29c4ca063 100644 --- a/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java +++ b/algo/src/test/java/org/neo4j/gds/triangle/TriangleStreamTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.graphbuilder.DefaultBuilder; import org.neo4j.gds.graphbuilder.GraphBuilder; import org.neo4j.graphdb.Node; @@ -73,7 +73,7 @@ void setupGraphDb() { void testSequential() { TripleConsumer mock = mock(TripleConsumer.class); - TriangleStream.create(graph, Pools.DEFAULT, 1) + TriangleStream.create(graph, DefaultPool.INSTANCE, 1) .compute() .forEach(r -> mock.consume(r.nodeA, r.nodeB, r.nodeC)); @@ -84,7 +84,7 @@ void testSequential() { void testParallel() { TripleConsumer mock = mock(TripleConsumer.class); - TriangleStream.create(graph, Pools.DEFAULT, 8) + TriangleStream.create(graph, DefaultPool.INSTANCE, 8) .compute() .forEach(r -> mock.consume(r.nodeA, r.nodeB, r.nodeC)); diff --git a/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java b/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java new file mode 100644 index 00000000000..1e71af7ac17 --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/triangle/UnionGraphTriangleCountingTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.triangle; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.Orientation; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.concurrency.DefaultPool; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +@GdlExtension +class UnionGraphTriangleCountingTest { + @GdlGraph(orientation = Orientation.UNDIRECTED) + static String DB_CYPHER="CREATE "+ + "(a0:L4)," + + "(a1:L1)," + + "(a2:L3)," + + "(a3:L1)," + + "(a5:L3)," + + "(a6:L1)," + + "(a7:L4)," + + "(a8:L3)," + + "(a9:L4)," + + "(a10:L1)," + + "(a11:L1)," + + "(a12:L4)," + + "(a13:L4)," + + "(a14:L3)," + + "(a15:L3)," + + "(a17:L4)," + + "(a18:L0)," + + "(a19:L3)," + + "(a20:L1)," + + "(a21:L1)," + + "(a22:L1)," + + "(a23:L2)," + + "(a24:L4)," + + "(a25:L3)," + + "(a26:L0)," + + "(a28:L4)," + + "(a29:L4)," + + "(a11)-[:T1]->(a15), "+ + "(a0)-[:T1]->(a21), "+ + "(a8)-[:T2]->(a28), "+ + "(a12)-[:T3]->(a12), "+ + "(a9)-[:T4]->(a10), "+ + "(a3)-[:T2]->(a26), "+ + "(a7)-[:T0]->(a21), "+ + "(a11)-[:T3]->(a29), "+ + "(a1)-[:T3]->(a14), "+ + "(a14)-[:T0]->(a22), "+ + "(a10)-[:T1]->(a13), "+ + "(a3)-[:T0]->(a21), "+ + "(a5)-[:T3]->(a28), "+ + "(a10)-[:T3]->(a25), "+ + "(a8)-[:T1]->(a14), "+ + "(a11)-[:T3]->(a15), "+ + "(a13)-[:T2]->(a18), "+ + "(a13)-[:T4]->(a20), "+ + "(a6)-[:T1]->(a29), "+ + "(a12)-[:T3]->(a14), "+ + "(a3)-[:T2]->(a21), "+ + "(a2)-[:T1]->(a21), "+ + "(a0)-[:T0]->(a20), "+ + "(a24)-[:T0]->(a29), "+ + "(a10)-[:T4]->(a19), "+ + "(a0)-[:T1]->(a28), "+ + "(a9)-[:T4]->(a17), "+ + "(a15)-[:T4]->(a21), "+ + "(a21)-[:T1]->(a24) "; + + @Inject + Graph graph; + + @Test + void shouldWorkWithUnionGraphs() { + var config=TriangleCountStreamConfigImpl.builder().concurrency(1).build(); + var a=IntersectingTriangleCount.create(graph,config, DefaultPool.INSTANCE); + var result=a.compute(); + assertThat(result.globalTriangles()).isEqualTo(0); + } +} diff --git a/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java b/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java index b09420ea94e..94ecc433dbc 100644 --- a/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java +++ b/algo/src/test/java/org/neo4j/gds/undirected/ToUndirectedTest.java @@ -28,7 +28,7 @@ import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -82,7 +82,7 @@ void shouldCreateUndirectedRelationships(int concurrency) { directedGraphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); directedGraphStore.addRelationshipType(undirectedRelationships); @@ -129,7 +129,7 @@ void shouldCreateUndirectedRelationshipsWithSingleRelationshipProperty(int concu singleDirectedGraphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); singleDirectedGraphStore.addRelationshipType(undirectedRelationships); @@ -175,7 +175,7 @@ void shouldCreateUndirectedRelationshipsWithNoRelationshipProperty(int concurren noPropertyDirectedGraphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); noPropertyDirectedGraphStore.addRelationshipType(undirectedRelationships); @@ -207,7 +207,7 @@ void shouldAggregateWithoutProperties() { inputGraphStore, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); inputGraphStore.addRelationshipType(undirectedRels); @@ -240,7 +240,7 @@ void shouldAggregateWithPropertiesAndGlobalAggregation() { input, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); input.addRelationshipType(aggregatedUndirectedRelationships); @@ -281,7 +281,7 @@ void shouldAggregateWithPropertiesAndLocalAggregation() { input, config, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); input.addRelationshipType(aggregatedUndirectedRelationships); diff --git a/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java b/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java index d5d3dfc3647..08fb482dda8 100644 --- a/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/walking/CollapseMultiPathsTest.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.AdjacencyList; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; @@ -112,7 +112,7 @@ void shouldFollowRoutesFromMovies() { false, RelationshipType.of("REL"), 2, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); @@ -174,7 +174,7 @@ void shouldTurnEveryRouteIntoRelationship() { false, RelationshipType.of("REL"), 2, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java b/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java index 933e43901d7..02a83910f84 100644 --- a/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java +++ b/algo/src/test/java/org/neo4j/gds/walking/CollapsePathTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -123,7 +123,7 @@ void testCreatingRelationships() { false, RelationshipType.of("SAME_DRUG"), 2, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); @@ -139,7 +139,7 @@ void testAllowCreatingSelfLoops() { true, RelationshipType.of("SAME_DRUG"), 2, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); @@ -153,7 +153,7 @@ void runWithDifferentRelationshipTypes() { false, RelationshipType.of("SAME_DRUG"), 2, - Pools.DEFAULT + DefaultPool.INSTANCE ).compute(); assertResultGraph(tookGraphStore, relationships, EXPECTED_WITHOUT_LOOPS); diff --git a/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java b/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java index 9ced3d34b69..65bd6e58e79 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/IncrementalWccTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.config.ConcurrencyConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -125,7 +125,7 @@ void shouldAssignMinimumCommunityIdOnMerge() { private DisjointSetStruct run(Graph graph, WccBaseConfig config) { return new Wcc( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, COMMUNITY_SIZE / ConcurrencyConfig.DEFAULT_CONCURRENCY, config, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java b/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java index 88cbde8ba0e..9c2402ee096 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/WccTest.java @@ -35,7 +35,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -280,7 +280,7 @@ DisjointSetStruct run(Graph graph, WccBaseConfig config) { DisjointSetStruct run(Graph graph, WccBaseConfig config, int concurrency) { return new Wcc( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, communitySize() / concurrency, config, ProgressTracker.NULL_TRACKER diff --git a/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java b/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java index 228b1ab852f..7b05ab90234 100644 --- a/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java +++ b/algo/src/test/java/org/neo4j/gds/wcc/WccThresholdTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.CommunityHelper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -110,7 +110,7 @@ private void assertResults(double threshold, TestGraph graph, String[][] expecte DisjointSetStruct dss = new Wcc( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, DEFAULT_BATCH_SIZE, wccConfig, ProgressTracker.NULL_TRACKER diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/pregel/Hits.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/pregel/Hits.java index 2dd1a02ea72..9f2e104898d 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/pregel/Hits.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/pregel/Hits.java @@ -171,7 +171,9 @@ default String authProperty() { return "auth"; } - @Override + @Value.Default + @Configuration.ConvertWith(method = "org.neo4j.gds.beta.pregel.Partitioning#parse") + @Configuration.ToMapValue("org.neo4j.gds.beta.pregel.Partitioning#toString") default Partitioning partitioning() { return Partitioning.AUTO; } diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsMutateProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsMutateProcTest.java new file mode 100644 index 00000000000..043113fc3e5 --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsMutateProcTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.hits; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.gds.pregel.HitsMutateProc; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class HitsMutateProcTest extends BaseProcTest { + + @Neo4jGraph + public static final String DB_CYPHER = + "CREATE" + + " (a:Website {name: 'A'})," + + " (b:Website {name: 'B'})," + + " (c:Website {name: 'C'})," + + " (d:Website {name: 'D'})," + + " (e:Website {name: 'E'})," + + " (f:Website {name: 'F'})," + + " (g:Website {name: 'G'})," + + " (h:Website {name: 'H'})," + + " (i:Website {name: 'I'})," + + + " (a)-[:LINK]->(b)," + + " (a)-[:LINK]->(c)," + + " (a)-[:LINK]->(d)," + + " (b)-[:LINK]->(c)," + + " (b)-[:LINK]->(d)," + + " (c)-[:LINK]->(d)," + + + " (e)-[:LINK]->(b)," + + " (e)-[:LINK]->(d)," + + " (e)-[:LINK]->(f)," + + " (e)-[:LINK]->(h)," + + + " (f)-[:LINK]->(g)," + + " (f)-[:LINK]->(i)," + + " (f)-[:LINK]->(h)," + + " (g)-[:LINK]->(h)," + + " (g)-[:LINK]->(i)," + + " (h)-[:LINK]->(i)"; + + @BeforeEach + void setupGraph() throws Exception { + registerProcedures( + HitsMutateProc.class, + GraphProjectProc.class + ); + + runQuery( + "CALL gds.graph.project(" + + " 'myGraph'," + + " 'Website'," + + " {LINK: {indexInverse: true}}" + + ");"); + } + + @Test + void shouldRunWithoutError() { + assertThatNoException() + .as("The `HITS` write procedure should run without raising an exception.") + .isThrownBy(() -> runQuery("CALL gds.alpha.hits.mutate('myGraph', { hitsIterations: 1, writeProperty:'hits' })")); + } +} diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsWriteProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsWriteProcTest.java new file mode 100644 index 00000000000..4e8e136fb02 --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/hits/HitsWriteProcTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.hits; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.gds.pregel.HitsWriteProc; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class HitsWriteProcTest extends BaseProcTest { + + @Neo4jGraph + public static final String DB_CYPHER = + "CREATE" + + " (a:Website {name: 'A'})," + + " (b:Website {name: 'B'})," + + " (c:Website {name: 'C'})," + + " (d:Website {name: 'D'})," + + " (e:Website {name: 'E'})," + + " (f:Website {name: 'F'})," + + " (g:Website {name: 'G'})," + + " (h:Website {name: 'H'})," + + " (i:Website {name: 'I'})," + + + " (a)-[:LINK]->(b)," + + " (a)-[:LINK]->(c)," + + " (a)-[:LINK]->(d)," + + " (b)-[:LINK]->(c)," + + " (b)-[:LINK]->(d)," + + " (c)-[:LINK]->(d)," + + + " (e)-[:LINK]->(b)," + + " (e)-[:LINK]->(d)," + + " (e)-[:LINK]->(f)," + + " (e)-[:LINK]->(h)," + + + " (f)-[:LINK]->(g)," + + " (f)-[:LINK]->(i)," + + " (f)-[:LINK]->(h)," + + " (g)-[:LINK]->(h)," + + " (g)-[:LINK]->(i)," + + " (h)-[:LINK]->(i)"; + + @BeforeEach + void setupGraph() throws Exception { + registerProcedures( + HitsWriteProc.class, + GraphProjectProc.class + ); + + runQuery( + "CALL gds.graph.project(" + + " 'myGraph'," + + " 'Website'," + + " {LINK: {indexInverse: true}}" + + ");"); + } + + @Test + void shouldRunWithoutError() { + assertThatNoException() + .as("The `HITS` mutate procedure should run without raising an exception.") + .isThrownBy(() -> runQuery("CALL gds.alpha.hits.write('myGraph', { hitsIterations: 1, mutateProperty:'hits' })")); + } +} diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java index 5dcd1057599..a68d12abb9d 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/HitsTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -65,7 +65,7 @@ void testHits() { graph, config, new Hits(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAMutateProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAMutateProcTest.java new file mode 100644 index 00000000000..bd0230b44e8 --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAMutateProcTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pregel; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.beta.generator.GraphGenerateProc; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class SpeakerListenerLPAMutateProcTest extends BaseProcTest { + + @BeforeEach + void setup() throws Exception { + registerProcedures( + SpeakerListenerLPAMutateProc.class, + GraphGenerateProc.class + ); + } + + @Test + void shouldNotFailWhenRunningOnNonWritableGraph() { + runQuery("CALL gds.beta.graph.generate('randomGraph', 5, 2, {relationshipSeed:19}) YIELD name, nodes, relationships, relationshipDistribution"); + + assertThatNoException().isThrownBy( + () -> runQuery("CALL gds.alpha.sllpa.mutate('randomGraph', {mutateProperty: 'm', maxIterations: 4, minAssociationStrength: 0.1})") + ); + } +} diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAStreamProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAStreamProcTest.java new file mode 100644 index 00000000000..b38e06f46b8 --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAStreamProcTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pregel; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.beta.generator.GraphGenerateProc; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class SpeakerListenerLPAStreamProcTest extends BaseProcTest { + + @BeforeEach + void setup() throws Exception { + registerProcedures( + SpeakerListenerLPAStreamProc.class, + GraphGenerateProc.class + ); + } + + @Test + void shouldNotFailWhenRunningOnNonWritableGraph() { + runQuery("CALL gds.beta.graph.generate('randomGraph', 5, 2, {relationshipSeed:19}) YIELD name, nodes, relationships, relationshipDistribution"); + + assertThatNoException().isThrownBy( + () -> runQuery("CALL gds.alpha.sllpa.stream('randomGraph', {maxIterations: 4, minAssociationStrength: 0.1})") + ); + } +} diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java index 30b5e70d348..4752a25b2a6 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPATest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -75,7 +75,7 @@ void testWithoutPruning() { graph, config, new SpeakerListenerLPA(42), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -122,7 +122,7 @@ void prunesAwayAfterManyIterations() { graph, config, new SpeakerListenerLPA(42), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -164,7 +164,7 @@ void closesThreadLocal() { graph, config, computation, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).run(); diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAWriteProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAWriteProcTest.java new file mode 100644 index 00000000000..8ade6371ef4 --- /dev/null +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/pregel/SpeakerListenerLPAWriteProcTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pregel; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.beta.generator.GraphGenerateProc; +import org.neo4j.graphdb.QueryExecutionException; + +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +class SpeakerListenerLPAWriteProcTest extends BaseProcTest { + + @BeforeEach + void setup() throws Exception { + registerProcedures( + SpeakerListenerLPAWriteProc.class, + GraphGenerateProc.class + ); + } + + @Test + void shouldFailWhenRunningOnNonWritableGraph() { + runQuery("CALL gds.beta.graph.generate('randomGraph', 5, 2, {relationshipSeed:19}) YIELD name, nodes, relationships, relationshipDistribution"); + + assertThatExceptionOfType(QueryExecutionException.class) + .isThrownBy( + () -> runQuery("CALL gds.alpha.sllpa.write('randomGraph', {writeProperty: 'm', maxIterations: 4, minAssociationStrength: 0.1})") + ) + .withRootCauseInstanceOf(IllegalArgumentException.class) + .withMessageContaining("The provided graph does not support `write` execution mode."); + } +} diff --git a/build.gradle b/build.gradle index 6d2713de31c..bc8e4fd1d09 100644 --- a/build.gradle +++ b/build.gradle @@ -35,6 +35,10 @@ ext { project(':neo4j-kernel-adapter-5.6'), project(':neo4j-kernel-adapter-5.7'), project(':neo4j-kernel-adapter-5.8'), + project(':neo4j-kernel-adapter-5.9'), + project(':neo4j-kernel-adapter-5.10'), + project(':neo4j-kernel-adapter-5.11'), + project(':neo4j-kernel-adapter-5.12'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -46,6 +50,10 @@ ext { project(':storage-engine-adapter-5.6'), project(':storage-engine-adapter-5.7'), project(':storage-engine-adapter-5.8'), + project(':storage-engine-adapter-5.9'), + project(':storage-engine-adapter-5.10'), + project(':storage-engine-adapter-5.11'), + project(':storage-engine-adapter-5.12'), ] ] } diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index 946ecd4b7e6..2d967bcf3e8 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -41,6 +41,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -84,6 +85,7 @@ import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -107,6 +109,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -137,8 +141,10 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -511,7 +517,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -803,4 +810,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService).name()); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java index e31769817d3..a82ceac1064 100644 --- a/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java +++ b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader44; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader44(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index d42c6df7c5b..30a659e7815 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -147,8 +151,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -605,7 +611,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -920,4 +927,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java index 532768751cd..df302f95b11 100644 --- a/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader51; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader51(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.10/neo4j-kernel-adapter/build.gradle b/compatibility/5.10/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..7b8bcc52246 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.10' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.10' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.10' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.10' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.10' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.10' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.10' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..119c57cae44 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.10 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.10 (placeholder)"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..9a92eb7c201 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.10 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.10 (placeholder)"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/BoltTransactionRunnerImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..c507ade6887 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableProcedureImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableProcedureImpl.java new file mode 100644 index 00000000000..b273fb43e8f --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableUserAggregationFunctionImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..d120d4b8314 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatAccessModeImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatAccessModeImpl.java new file mode 100644 index 00000000000..2152447a12b --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatGraphDatabaseAPIImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..c7c19eda5ed --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatIndexQueryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..35211b88555 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatUsernameAuthSubjectImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..d4521de6ed4 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompositeNodeCursorImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..95fab00f468 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseLayoutImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..e656e8bc798 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..3fd29287f7a --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..f60c8eda563 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_10; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.10"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java new file mode 100644 index 00000000000..63b70196011 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/Neo4jProxyImpl.java @@ -0,0 +1,968 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials, + threadSafe + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/NodeLabelIndexLookupImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..aaf6b0ee05e --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/PartitionedStoreScan.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/PartitionedStoreScan.java new file mode 100644 index 00000000000..5d542426fd6 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ReferencePropertyReference.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ReferencePropertyReference.java new file mode 100644 index 00000000000..c94d0d621dd --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ScanBasedStoreScanImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..f26affc5f05 --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..69239b3098d --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_10; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.10"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyImpl.java new file mode 100644 index 00000000000..26ca820cefc --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/TestLogImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/TestLogImpl.java new file mode 100644 index 00000000000..cd67346d60c --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/VirtualRelationshipImpl.java b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..d7a9c41076f --- /dev/null +++ b/compatibility/5.10/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_510/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/build.gradle b/compatibility/5.10/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..4ab26345d26 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.10' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.10' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.10' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.10' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.10' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.10' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java b/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..ec941c14b1e --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported510"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..6267c704fec --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.10 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.10"; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCommandCreationContextImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..74582802065 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCountsStoreImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..1d31cd83d7a --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryMetaDataProviderImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..884eff2c379 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository510; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository510 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository510(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodeCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodeCursor.java new file mode 100644 index 00000000000..3b1e13f7184 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodePropertyCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..41133f54533 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertyCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..2478a7ca9f5 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertySelectionImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..d48bf99cb54 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipPropertyCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..dc4ad73301e --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipScanCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..7ab410e6661 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipTraversalCursor.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..19d93b86a4c --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..e1c2e09a95c --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineFactory.java @@ -0,0 +1,568 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory510; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccessExtended; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-510"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_10, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @Override + public SchemaRuleMigrationAccessExtended schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + MemoryTracker memoryTracker + ) throws IOException { + return null; // Store copy that uses this method doesn't accept GDS in memory store formats target + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory510.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock clock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..484f0c1a19a --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageEngineImpl.java @@ -0,0 +1,347 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader510; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.InternalErrorTracer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + throw new UnsupportedOperationException(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader510(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory(StorageEngineFactory storageEngineFactory, Config config, SystemNanoClock clock) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } + + @Override + public InternalErrorTracer internalErrorTracer() { + return InternalErrorTracer.NO_TRACER; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageLocksImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..bbc08d7e808 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStoreVersion.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStoreVersion.java new file mode 100644 index 00000000000..669b0bf4983 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryStoreVersion.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.configuration.Config; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion(Config config) { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } + +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryTransactionIdStoreImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..2a7a57be260 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryVersionCheck.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryVersionCheck.java new file mode 100644 index 00000000000..8f32317048b --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._510.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..9f5c42c0f5d --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_10; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.10"; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..0e4e3e89cf4 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_510/StorageEngineProxyImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._510; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository510.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository510.java new file mode 100644 index 00000000000..35b0e6c8dba --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository510.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository510 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository510() { + this(0, 0); + } + + private InMemoryLogVersionRepository510(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory510.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory510.java new file mode 100644 index 00000000000..710f94121eb --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory510.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory510 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory510(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader510.java b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader510.java new file mode 100644 index 00000000000..7f5095a1520 --- /dev/null +++ b/compatibility/5.10/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader510.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._510.InMemoryNodeCursor; +import org.neo4j.gds.compat._510.InMemoryPropertyCursor; +import org.neo4j.gds.compat._510.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._510.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader510 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader510( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/build.gradle b/compatibility/5.11/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..927eee8f07c --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.11' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.11' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.11' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.11' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.11' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.11' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.11' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..32ae5e87d89 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.11 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.11 (placeholder)"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..2e0c1a9e2b7 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.11 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.11 (placeholder)"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/BoltTransactionRunnerImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..f76c6382881 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableProcedureImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableProcedureImpl.java new file mode 100644 index 00000000000..ac60ee006bd --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableUserAggregationFunctionImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..b7a79d76f37 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java new file mode 100644 index 00000000000..226f56cd1bb --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatGraphDatabaseAPIImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..a4c2fc7a6de --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatIndexQueryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..11d2514b955 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatUsernameAuthSubjectImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..79a8b581efd --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompositeNodeCursorImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..98978df157a --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseLayoutImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..1dda86e46e0 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..665ae3d6cda --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..7719f97a641 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_11; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.11"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java new file mode 100644 index 00000000000..05a406402a8 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/Neo4jProxyImpl.java @@ -0,0 +1,986 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.TokenSet; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.ReadSecurityPropertyProvider; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode) { + @Override + public boolean allowsReadNodeProperty( + Supplier labels, + int propertyKey, + ReadSecurityPropertyProvider propertyProvider + ) { + return custom.allowsReadNodeProperty(propertyKey); + } + }; + } + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials, + threadSafe + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format( + Locale.ENGLISH, + "The read version string length %d is not proper.", + length + )); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency( + ctx.dependencyResolver(), + GlobalProcedures.class + ); + return globalProcedures.getCurrentView().lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getCurrentView().getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/NodeLabelIndexLookupImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..5465d3261a2 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/PartitionedStoreScan.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/PartitionedStoreScan.java new file mode 100644 index 00000000000..cbd2f63fcf5 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ReferencePropertyReference.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ReferencePropertyReference.java new file mode 100644 index 00000000000..7ec85a1d19c --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ScanBasedStoreScanImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..fbee3ca33b1 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..0782eebff8d --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_11; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.11"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyImpl.java new file mode 100644 index 00000000000..965caa2d66d --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/TestLogImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/TestLogImpl.java new file mode 100644 index 00000000000..cbacd42e0d4 --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/VirtualRelationshipImpl.java b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..e14ab2d222e --- /dev/null +++ b/compatibility/5.11/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_511/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/build.gradle b/compatibility/5.11/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..0a21467ee37 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.11' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.11' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.11' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.11' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.11' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.11' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java b/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..6b85ad71504 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported511"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..6d2f3217423 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.11 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.11"; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCommandCreationContextImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..da81b6bbe34 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCountsStoreImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..6e3814395cd --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryCountsStoreImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsStore; +import org.neo4j.counts.CountsUpdater; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +import java.io.IOException; + +public class InMemoryCountsStoreImpl implements CountsStore { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start(CursorContext cursorContext, MemoryTracker memoryTracker) throws IOException { + + } + + @Override + public CountsUpdater updater(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory cursorContextFactory, + int i, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryMetaDataProviderImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..23790cccb49 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository511; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository511 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository511(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodeCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodeCursor.java new file mode 100644 index 00000000000..5f4bb3cdd0b --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodePropertyCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..4463339e2b6 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertyCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..502fda65b2b --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertySelectionImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..839a0b7fb34 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipPropertyCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..43d155aba61 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipScanCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..f64ac350f16 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipTraversalCursor.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..7be8a436257 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..d08d8b633a3 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineFactory.java @@ -0,0 +1,571 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory511; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccessExtended; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-511"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_11, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) throws IOException { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( + NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME + ) + ) { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + @Override + public SchemaRuleMigrationAccessExtended schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + MemoryTracker memoryTracker + ) { + // this is used by store copy, which is not supported for GDS storage engine + return null; + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + try ( + var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors(stores, cursorContext) + ) { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory511.INSTANCE; + } + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog reportLog, + InternalLog verboseLog, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier throwingSupplier, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock systemNanoClock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..7a80d4e67b0 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageEngineImpl.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsStore; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader511; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.fs.WritableChannel; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.InternalErrorTracer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader511(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsStore countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext(boolean multiVersioned) { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory( + StorageEngineFactory storageEngineFactory, + Config config, + SystemNanoClock systemNanoClock + ) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } + + @Override + public void preAllocateStoreFilesForCommands(CommandBatchToApply batch, TransactionApplicationMode mode) { + // GDS has its own mechanism of memory allocation, so we don't need this + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + return new EnrichmentCommand() { + + @Override + public Enrichment enrichment() { + return null; + } + + @Override + public void serialize(WritableChannel channel) { + + } + + @Override + public KernelVersion kernelVersion() { + return kernelVersion; + } + }; + } + + @Override + public InternalErrorTracer internalErrorTracer() { + return InternalErrorTracer.NO_TRACER; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageLocksImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..a7ac028ce8f --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStoreVersion.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStoreVersion.java new file mode 100644 index 00000000000..946e7579069 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryStoreVersion.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.configuration.Config; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion(Config config) { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } + +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryTransactionIdStoreImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..c95a6524be9 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryVersionCheck.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryVersionCheck.java new file mode 100644 index 00000000000..8cb999f8c76 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._511.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..687468d4e45 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_11; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.11"; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyImpl.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..d2688f8a37a --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_511/StorageEngineProxyImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._511; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository511.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository511.java new file mode 100644 index 00000000000..542a9239cdb --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository511.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository511 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository511() { + this(0, 0); + } + + private InMemoryLogVersionRepository511(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory511.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory511.java new file mode 100644 index 00000000000..004ffea98d4 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory511.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory511 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory511(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader511.java b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader511.java new file mode 100644 index 00000000000..9ba49899c70 --- /dev/null +++ b/compatibility/5.11/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader511.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsStore; +import org.neo4j.gds.compat._511.InMemoryNodeCursor; +import org.neo4j.gds.compat._511.InMemoryPropertyCursor; +import org.neo4j.gds.compat._511.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._511.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader511 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsStore counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader511( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsStore counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .unionProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .unionProperties() + .size(); + + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().entries().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/build.gradle b/compatibility/5.12/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..498b905b8ba --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.12' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.12' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.12' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.12' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.12' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.12' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.12' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..3c34c8cd8d5 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.12 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.12 (placeholder)"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..2c165303d66 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.12 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.12 (placeholder)"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/BoltTransactionRunnerImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..6148b17ef9d --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableProcedureImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableProcedureImpl.java new file mode 100644 index 00000000000..c940486e6d2 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableUserAggregationFunctionImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..68e4cd838a1 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java new file mode 100644 index 00000000000..f11614a613b --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatGraphDatabaseAPIImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..f4995ac8ba4 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatIndexQueryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..143ccbdbdb2 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatUsernameAuthSubjectImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..67ffd47576d --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompositeNodeCursorImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..c63636a254d --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseLayoutImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..a013d2ed0d6 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..d1556f00eac --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..4b7a863afd7 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_12; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.12"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java new file mode 100644 index 00000000000..73752765f09 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/Neo4jProxyImpl.java @@ -0,0 +1,978 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.TokenSet; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.ReadSecurityPropertyProvider; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.FixedVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.BitBuffer; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode) { + @Override + public boolean allowsReadNodeProperty( + Supplier labels, + int propertyKey, + ReadSecurityPropertyProvider propertyProvider + ) { + return custom.allowsReadNodeProperty(propertyKey); + } + }; + } + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, FixedVersionContextSupplier.EMPTY_CONTEXT_SUPPLIER) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials, + threadSafe + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + BitBuffer bits = BitBuffer.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, FixedVersionContextSupplier.EMPTY_CONTEXT_SUPPLIER) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.getCurrentView().lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getCurrentView().getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/NodeLabelIndexLookupImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..58d54c38069 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/PartitionedStoreScan.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/PartitionedStoreScan.java new file mode 100644 index 00000000000..365e85330d6 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ReferencePropertyReference.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ReferencePropertyReference.java new file mode 100644 index 00000000000..ee12df9ca66 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ScanBasedStoreScanImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..9319f2f4234 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..52e994f8e91 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_12; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.12"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyImpl.java new file mode 100644 index 00000000000..8ef4811ea9e --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/TestLogImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/TestLogImpl.java new file mode 100644 index 00000000000..ed2748fc962 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/VirtualRelationshipImpl.java b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..938801ad806 --- /dev/null +++ b/compatibility/5.12/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_512/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/build.gradle b/compatibility/5.12/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..29c27207344 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.12' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.12' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.12' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.12' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.12' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.12' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..0c60e8ffa8e --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported512"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..9ac73ba4f3d --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.12 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.12"; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCommandCreationContextImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..b881db2eabf --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCountsStoreImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..217b653238a --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryCountsStoreImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsStore; +import org.neo4j.counts.CountsUpdater; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +import java.io.IOException; + +public class InMemoryCountsStoreImpl implements CountsStore { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start(CursorContext cursorContext, MemoryTracker memoryTracker) throws IOException { + + } + + @Override + public CountsUpdater updater(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory cursorContextFactory, + int i, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryMetaDataProviderImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..43cb7ca0cf0 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository512; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository512 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository512(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodeCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodeCursor.java new file mode 100644 index 00000000000..08723864755 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodePropertyCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..b766d0cbc3b --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertyCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..4522abddbb9 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertySelectionImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..7cbbf1f0589 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipPropertyCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..059d09ea714 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipScanCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..b4d226ec194 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipTraversalCursor.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..197ee873aa1 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..07f04f4285d --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineFactory.java @@ -0,0 +1,571 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory512; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccessExtended; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-512"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_12, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) throws IOException { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( + NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME + ) + ) { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + @Override + public SchemaRuleMigrationAccessExtended schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + MemoryTracker memoryTracker + ) { + // this is used by store copy, which is not supported for GDS storage engine + return null; + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + try ( + var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors(stores, cursorContext) + ) { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory512.INSTANCE; + } + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog reportLog, + InternalLog verboseLog, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier throwingSupplier, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock systemNanoClock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..0309c9f476d --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageEngineImpl.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsStore; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader512; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.fs.WritableChannel; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.InternalErrorTracer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader512(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsStore countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext(boolean multiVersioned) { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory( + StorageEngineFactory storageEngineFactory, + Config config, + SystemNanoClock systemNanoClock + ) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } + + @Override + public void preAllocateStoreFilesForCommands(CommandBatchToApply batch, TransactionApplicationMode mode) { + // GDS has its own mechanism of memory allocation, so we don't need this + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + return new EnrichmentCommand() { + + @Override + public Enrichment enrichment() { + return null; + } + + @Override + public void serialize(WritableChannel channel) { + + } + + @Override + public KernelVersion kernelVersion() { + return kernelVersion; + } + }; + } + + @Override + public InternalErrorTracer internalErrorTracer() { + return InternalErrorTracer.NO_TRACER; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageLocksImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..e31f564272d --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStoreVersion.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStoreVersion.java new file mode 100644 index 00000000000..a587e43619f --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryStoreVersion.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.configuration.Config; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion(Config config) { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } + +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryTransactionIdStoreImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..db17b078973 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryVersionCheck.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryVersionCheck.java new file mode 100644 index 00000000000..c23fab7aeb2 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._512.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..3b0a804d07f --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_12; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.12"; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyImpl.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..9869947a408 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_512/StorageEngineProxyImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._512; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository512.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository512.java new file mode 100644 index 00000000000..16023f54e43 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository512.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository512 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository512() { + this(0, 0); + } + + private InMemoryLogVersionRepository512(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory512.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory512.java new file mode 100644 index 00000000000..1e59fae310d --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory512.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory512 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory512(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader512.java b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader512.java new file mode 100644 index 00000000000..9a67420e604 --- /dev/null +++ b/compatibility/5.12/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader512.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsStore; +import org.neo4j.gds.compat._512.InMemoryNodeCursor; +import org.neo4j.gds.compat._512.InMemoryPropertyCursor; +import org.neo4j.gds.compat._512.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._512.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader512 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsStore counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader512( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsStore counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .unionProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .unionProperties() + .size(); + + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().entries().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/build.gradle b/compatibility/5.13/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..a7828abf67b --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.13' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.13' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.13' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.13' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.13' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.13' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.13' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..668351e0ed0 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.13 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.13 (placeholder)"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..5abd5f6fcc0 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.13 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.13 (placeholder)"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/BoltTransactionRunnerImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..1c9014edbf6 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableProcedureImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableProcedureImpl.java new file mode 100644 index 00000000000..ed8ee50b240 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableUserAggregationFunctionImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..3c51894b2ed --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java new file mode 100644 index 00000000000..568f3c7d0ce --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatGraphDatabaseAPIImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..0cbc8185280 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatIndexQueryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..9265647abb5 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatUsernameAuthSubjectImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..090434df0d8 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompositeNodeCursorImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..7a12da5dfc0 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseLayoutImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..0f2d80d043f --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..c35f5b69d20 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..a66b1469ed5 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_13; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.13"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java new file mode 100644 index 00000000000..c3a0eee1a32 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/Neo4jProxyImpl.java @@ -0,0 +1,978 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.TokenSet; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.ReadSecurityPropertyProvider; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.FixedVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.BitBuffer; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode) { + @Override + public boolean allowsReadNodeProperty( + Supplier labels, + int propertyKey, + ReadSecurityPropertyProvider propertyProvider + ) { + return custom.allowsReadNodeProperty(propertyKey); + } + }; + } + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, FixedVersionContextSupplier.EMPTY_CONTEXT_SUPPLIER) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials, + threadSafe + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + BitBuffer bits = BitBuffer.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, FixedVersionContextSupplier.EMPTY_CONTEXT_SUPPLIER) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.getCurrentView().lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getCurrentView().getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getCurrentView().getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/NodeLabelIndexLookupImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..23bd3ab65be --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/PartitionedStoreScan.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/PartitionedStoreScan.java new file mode 100644 index 00000000000..12e483dea59 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ReferencePropertyReference.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ReferencePropertyReference.java new file mode 100644 index 00000000000..93175c637a9 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ScanBasedStoreScanImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..61254cbe91d --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..ed12846cabc --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_13; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.13"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyImpl.java new file mode 100644 index 00000000000..7dd33b8b544 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/TestLogImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/TestLogImpl.java new file mode 100644 index 00000000000..b6f3c9c50c6 --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/VirtualRelationshipImpl.java b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..25372741c2b --- /dev/null +++ b/compatibility/5.13/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_513/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/build.gradle b/compatibility/5.13/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..7326e65721a --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.13' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.13' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.13' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.13' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.13' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.13' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java b/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..b2d569c80c4 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported513"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..85fbf8d91c6 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.13 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.13"; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCommandCreationContextImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..1662f5761ed --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCountsStoreImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..149e6c2458d --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryCountsStoreImpl.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsStore; +import org.neo4j.counts.CountsUpdater; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +import java.io.IOException; + +public class InMemoryCountsStoreImpl implements CountsStore { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start(CursorContext cursorContext, MemoryTracker memoryTracker) throws IOException { + + } + + @Override + public CountsUpdater updater(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory cursorContextFactory, + int i, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryMetaDataProviderImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..48585f772e0 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository513; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository513 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository513(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodeCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodeCursor.java new file mode 100644 index 00000000000..02c848b6e44 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodePropertyCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..63ef42897f4 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertyCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..a44b5cbaab3 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertySelectionImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..60e592442c7 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipPropertyCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..6571e0187ab --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipScanCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..2e6afb679ba --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipTraversalCursor.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..41a33a5f44d --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..7c098e6f198 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineFactory.java @@ -0,0 +1,572 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory513; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccessExtended; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-513"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_13, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) throws IOException { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( + NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME + ) + ) { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + @Override + public SchemaRuleMigrationAccessExtended schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + MemoryTracker memoryTracker, + LogTailMetadata logTail + ) { + // this is used by store copy, which is not supported for GDS storage engine + return null; + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + try ( + var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors(stores, cursorContext) + ) { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory513.INSTANCE; + } + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog reportLog, + InternalLog verboseLog, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier throwingSupplier, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock systemNanoClock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..2f3769728ed --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageEngineImpl.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsStore; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader513; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.fs.WritableChannel; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.InternalErrorTracer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader513(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsStore countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext(boolean multiVersioned) { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory( + StorageEngineFactory storageEngineFactory, + Config config, + SystemNanoClock systemNanoClock + ) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } + + @Override + public void preAllocateStoreFilesForCommands(CommandBatchToApply batch, TransactionApplicationMode mode) { + // GDS has its own mechanism of memory allocation, so we don't need this + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + return new EnrichmentCommand() { + + @Override + public Enrichment enrichment() { + return null; + } + + @Override + public void serialize(WritableChannel channel) { + + } + + @Override + public KernelVersion kernelVersion() { + return kernelVersion; + } + }; + } + + @Override + public InternalErrorTracer internalErrorTracer() { + return InternalErrorTracer.NO_TRACER; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageLocksImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..e09fd7303b1 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStoreVersion.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStoreVersion.java new file mode 100644 index 00000000000..1c1234b2deb --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryStoreVersion.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.configuration.Config; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion(Config config) { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } + +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryTransactionIdStoreImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..61f5ecac5ab --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryVersionCheck.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryVersionCheck.java new file mode 100644 index 00000000000..212a8717b2f --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/InMemoryVersionCheck.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._513.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public StoreVersionIdentifier getCurrentVersion(CursorContext cursorContext) throws + IllegalArgumentException, IllegalStateException { + return STORE_IDENTIFIER; + } + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public boolean isStoreVersionFullySupported(StoreVersionIdentifier storeVersion, CursorContext cursorContext) { + return true; // because we don't want to block upgrades + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..f3c7c98d3df --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_13; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.13"; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyImpl.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..6b342c12636 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_513/StorageEngineProxyImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._513; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository513.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository513.java new file mode 100644 index 00000000000..91a0e775497 --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository513.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository513 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository513() { + this(0, 0); + } + + private InMemoryLogVersionRepository513(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory513.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory513.java new file mode 100644 index 00000000000..7e238b01fee --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory513.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory513 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory513(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader513.java b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader513.java new file mode 100644 index 00000000000..602069e87bf --- /dev/null +++ b/compatibility/5.13/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader513.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsStore; +import org.neo4j.gds.compat._513.InMemoryNodeCursor; +import org.neo4j.gds.compat._513.InMemoryPropertyCursor; +import org.neo4j.gds.compat._513.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._513.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader513 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsStore counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader513( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsStore counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .unionProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .unionProperties() + .size(); + + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().entries().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index 017e4b313b1..9cfa9eda30e 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -147,8 +151,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -603,7 +609,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -918,4 +925,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java index 96a86ce2d35..df4a47e6279 100644 --- a/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader52; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader52(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index 994098ea39c..ee1a680d136 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -147,8 +151,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -603,7 +609,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -919,4 +926,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java index 08f7830ad65..fd1fd03c19a 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader53; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader53(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index 4a74f5600ad..7161bde2b53 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -147,8 +151,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -603,7 +609,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -918,4 +925,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java index f99cb355377..19c401d0882 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader54; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader54(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java index 978f9fe60b8..2a1172c96cc 100644 --- a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -604,7 +610,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -919,4 +926,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, DependencyResolver.SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java index 7124a0829d1..ae6f1c8df95 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader55; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader55(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java index 9e67bb68ab5..3b63c5ed99d 100644 --- a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -604,7 +610,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -919,4 +926,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java index b5a1b5a4ffc..f0c961de559 100644 --- a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader56; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader56(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java index 05c7cef633d..1a6813cce56 100644 --- a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -599,7 +605,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -914,4 +921,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java index 8a0eadab4b5..c845bc1451a 100644 --- a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader57; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader57(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java index fa8898dc9f9..ca9671f11d4 100644 --- a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java @@ -44,6 +44,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; @@ -87,6 +88,7 @@ import org.neo4j.internal.kernel.api.Scan; import org.neo4j.internal.kernel.api.TokenPredicate; import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -113,6 +115,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; @@ -148,8 +152,10 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.lang.String.format; import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; @@ -599,7 +605,8 @@ public ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return new ProcedureSignature( name, @@ -914,4 +921,47 @@ public boolean isCompositeDatabase(GraphDatabaseService databaseService) { var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } } diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java index c47d09fd394..fd8e08e5a39 100644 --- a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java @@ -22,7 +22,6 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; import org.neo4j.configuration.GraphDatabaseInternalSettings; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; @@ -35,15 +34,12 @@ import org.neo4j.graphdb.Direction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; -import org.neo4j.internal.recordstorage.InMemoryStorageReader58; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.storageengine.api.RelationshipSelection; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,11 +47,6 @@ public class StorageEngineProxyImpl implements StorageEngineProxyApi { - @Override - public CommandCreationContext inMemoryCommandCreationContext() { - return new InMemoryCommandCreationContextImpl(); - } - @Override public void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, @@ -69,13 +60,6 @@ public void initRelationshipTraversalCursorForRelType( cursor.init(sourceNodeId, -1, relationshipSelection); } - @Override - public StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return new InMemoryStorageReader58(graphStore, tokenHolders, counts); - } - @Override public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); diff --git a/compatibility/5.9/neo4j-kernel-adapter/build.gradle b/compatibility/5.9/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..d712c105351 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.9' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.9' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + compileOnly project(':annotations') + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.9' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..c972ffc94a3 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.9 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.9 (placeholder)"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..1a05f4b02fd --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.9 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.9 (placeholder)"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..479f8dbf179 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java new file mode 100644 index 00000000000..08ff067f966 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..de837dc4b2e --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java new file mode 100644 index 00000000000..49f92fb86e5 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..4fbf4bafb57 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..ccef737c591 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..85c75a5d377 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..3fc8fbfca62 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..01a2134ad14 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..e8e8185129c --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..25ad1de6d4e --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.9"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java new file mode 100644 index 00000000000..ce739eda4ef --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java @@ -0,0 +1,967 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GlobalProcedureRegistry; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.TextArray; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; +import org.neo4j.values.virtual.NodeValue; +import org.neo4j.values.virtual.VirtualValues; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials, + boolean threadSafe + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public NodeValue nodeValue(long id, TextArray labels, MapValue properties) { + return VirtualValues.nodeValue(id, String.valueOf(id), labels, properties); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe, + Optional deprecatedBy + ) { + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecatedBy.orElse(null), + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } + + @Override + public T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), GlobalProcedures.class); + return globalProcedures.lookupComponentProvider(component, safe).apply(ctx); + } + + @Override + public GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return new GlobalProcedureRegistry() { + @Override + public Set getAllProcedures() { + return globalProcedures.getAllProcedures(); + } + + @Override + public Stream getAllNonAggregatingFunctions() { + return globalProcedures.getAllNonAggregatingFunctions(); + } + + @Override + public Stream getAllAggregatingFunctions() { + return globalProcedures.getAllAggregatingFunctions(); + } + }; + } + + private static final DependencyResolver EMPTY_DEPENDENCY_RESOLVER = new DependencyResolver.Adapter() { + @Override + public T resolveDependency(Class type, SelectionStrategy selector) { + return null; + } + + @Override + public boolean containsDependency(Class type) { + return false; + } + }; + + @Override + public DependencyResolver emptyDependencyResolver() { + return EMPTY_DEPENDENCY_RESOLVER; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..849d9a4d5cd --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java new file mode 100644 index 00000000000..2a8da868aeb --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java new file mode 100644 index 00000000000..a24847fc2de --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..c108e8bec61 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..c39ad8aa279 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.9"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java new file mode 100644 index 00000000000..662daf64e73 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java new file mode 100644 index 00000000000..7d09c9ce756 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..07708b93b25 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/build.gradle b/compatibility/5.9/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..501dac746d1 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/build.gradle @@ -0,0 +1,68 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.9' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + compileOnly project(':annotations') + compileOnly project(':progress-tracking') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + java17CompileOnly project(':annotations') + java17CompileOnly project(':progress-tracking') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..4bdc7bd36f8 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported59"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..e43773a19d3 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.9"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..6bad2296b00 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..7cf4f836b7d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..77f9b72465f --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository59; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository59 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository59(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java new file mode 100644 index 00000000000..15d394b1586 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..43b8fd4837a --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..2b6499a1f96 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..5724e2829d3 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..806f0b25659 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..6f0c089ee8d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..6cd7ebf0310 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..95bd036fb06 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java @@ -0,0 +1,554 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineFactoryIdProvider; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory59; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-59"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_9, StorageEngineFactory.class); + } + + @Override + public byte id() { + return StorageEngineFactoryIdProvider.ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory59.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock clock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..e52651080d1 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java @@ -0,0 +1,341 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader59; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + throw new UnsupportedOperationException(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader59(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory(StorageEngineFactory storageEngineFactory, Config config, SystemNanoClock clock) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..0b49cdb34aa --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java new file mode 100644 index 00000000000..fcd75bd3438 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..2e220e4d3bc --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java new file mode 100644 index 00000000000..fb196c18fbe --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._59.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..81132c0c9de --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.9"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..81318ea3973 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java new file mode 100644 index 00000000000..793eae1cabd --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository59 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository59() { + this(0, 0); + } + + private InMemoryLogVersionRepository59(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java new file mode 100644 index 00000000000..4aac7460e26 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory59 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory59(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java new file mode 100644 index 00000000000..ae8687d07ab --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._59.InMemoryNodeCursor; +import org.neo4j.gds.compat._59.InMemoryPropertyCursor; +import org.neo4j.gds.compat._59.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._59.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader59 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader59( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().containsOriginalId(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GdsVersionInfoProvider.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GdsVersionInfoProvider.java new file mode 100644 index 00000000000..5c47cf293b6 --- /dev/null +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GdsVersionInfoProvider.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +public final class GdsVersionInfoProvider { + + private GdsVersionInfoProvider() {} + + public static final ProxyUtil.GdsVersionInfo GDS_VERSION_INFO = loadGdsVersion(); + + private static ProxyUtil.GdsVersionInfo loadGdsVersion() { + var builder = ImmutableGdsVersionInfo.builder(); + try { + // The class that we use to get the GDS version lives in proc-sysinfo, which is part of the released GDS jar, + // but we don't want to depend on that here. One reason is that this class gets generated and re-generated + // on every build and having it at the top of the dependency graph would cause a lot of recompilation. + // Let's do a bit of class loading and reflection to get the version. + var lookup = MethodHandles.lookup(); + + var buildInfoPropertiesClass = Class.forName("org.neo4j.gds.BuildInfoProperties"); + + // equivalent to: BuildInfoProperties.get() + var buildInfoPropertiesHandle = lookup.findStatic( + buildInfoPropertiesClass, + "get", + MethodType.methodType(buildInfoPropertiesClass) + ); + + // equivalent to: buildInfoProperties.gdsVersion() + var gdsVersionHandle = lookup.findVirtual( + buildInfoPropertiesClass, + "gdsVersion", + MethodType.methodType(String.class) + ); + + // var buildInfoProperties = BuildInfoProperties.get() + var buildInfoProperties = buildInfoPropertiesHandle.invoke(); + // var gdsVersion = buildInfoProperties.gdsVersion() + var gdsVersion = gdsVersionHandle.invoke(buildInfoProperties); + + return builder + .gdsVersion(String.valueOf(gdsVersion)) + .build(); + } catch (ClassNotFoundException e) { + builder.error(ImmutableErrorInfo.builder() + .logLevel(ProxyUtil.LogLevel.DEBUG) + .message( + "Could not determine GDS version, BuildInfoProperties is missing. " + + "This is likely due to not running GDS as a plugin, " + + "for example when running tests or using GDS as a Java module dependency." + ) + .reason(e) + .build() + ); + } catch (NoSuchMethodException | IllegalAccessException e) { + builder.error(ImmutableErrorInfo.builder() + .logLevel(ProxyUtil.LogLevel.WARN) + .message( + "Could not determine GDS version, the according methods on BuildInfoProperties could not be found.") + .reason(e) + .build() + ); + } catch (Throwable e) { + builder.error(ImmutableErrorInfo.builder() + .logLevel(ProxyUtil.LogLevel.WARN) + .message("Could not determine GDS version, the according methods on BuildInfoProperties failed.") + .reason(e) + .build() + ); + } + + return builder.gdsVersion("Unknown").build(); + } + +} diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GlobalProcedureRegistry.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GlobalProcedureRegistry.java new file mode 100644 index 00000000000..fd22cf2c8e2 --- /dev/null +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/GlobalProcedureRegistry.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat; + +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; + +import java.util.Set; +import java.util.stream.Stream; + +public interface GlobalProcedureRegistry { + Set getAllProcedures(); + Stream getAllNonAggregatingFunctions(); + Stream getAllAggregatingFunctions(); +} diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index 9e4b7547046..a15863019d1 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; +import org.neo4j.common.DependencyResolver; import org.neo4j.configuration.Config; import org.neo4j.configuration.connectors.ConnectorPortRegister; import org.neo4j.dbms.api.DatabaseManagementService; @@ -52,6 +53,7 @@ import org.neo4j.internal.kernel.api.Read; import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -72,6 +74,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.query.TransactionalContext; import org.neo4j.kernel.impl.query.TransactionalContextFactory; @@ -230,7 +234,8 @@ ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ); long getHighestPossibleNodeCount(Read read, @Nullable IdGeneratorFactory idGeneratorFactory); @@ -319,4 +324,10 @@ TransactionalContext newQueryContext( ); boolean isCompositeDatabase(GraphDatabaseService databaseService); + + T lookupComponentProvider(Context ctx, Class component, boolean safe) throws ProcedureException; + + GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures); + + DependencyResolver emptyDependencyResolver(); } diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java index bdc9ce856f7..0b291522965 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java @@ -25,8 +25,6 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.logging.Log; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.util.ArrayList; import java.util.Collection; import java.util.Locale; @@ -147,7 +145,7 @@ private static > ProxyInfobuilder() .factoryType(factoryClass) .neo4jVersion(NEO4J_VERSION_INFO) - .gdsVersion(GDS_VERSION_INFO) + .gdsVersion(GdsVersionInfoProvider.GDS_VERSION_INFO) .javaInfo(JAVA_INFO); try { @@ -191,77 +189,10 @@ private static Neo4jVersionInfo loadNeo4jVersion() { .reason(e) .build() ) - .neo4jVersion(Neo4jVersion.V_Dev) .build(); } } - public static final GdsVersionInfo GDS_VERSION_INFO = loadGdsVersion(); - - private static GdsVersionInfo loadGdsVersion() { - var builder = ImmutableGdsVersionInfo.builder(); - try { - // The class that we use to get the GDS version lives in proc-sysinfo, which is part of the released GDS jar, - // but we don't want to depend on that here. One reason is that this class gets generated and re-generated - // on every build and having it at the top of the dependency graph would cause a lot of recompilation. - // Let's do a bit of class loading and reflection to get the version. - var lookup = MethodHandles.lookup(); - - var buildInfoPropertiesClass = Class.forName("org.neo4j.gds.BuildInfoProperties"); - - // equivalent to: BuildInfoProperties.get() - var buildInfoPropertiesHandle = lookup.findStatic( - buildInfoPropertiesClass, - "get", - MethodType.methodType(buildInfoPropertiesClass) - ); - - // equivalent to: buildInfoProperties.gdsVersion() - var gdsVersionHandle = lookup.findVirtual( - buildInfoPropertiesClass, - "gdsVersion", - MethodType.methodType(String.class) - ); - - // var buildInfoProperties = BuildInfoProperties.get() - var buildInfoProperties = buildInfoPropertiesHandle.invoke(); - // var gdsVersion = buildInfoProperties.gdsVersion() - var gdsVersion = gdsVersionHandle.invoke(buildInfoProperties); - - return builder - .gdsVersion(String.valueOf(gdsVersion)) - .build(); - } catch (ClassNotFoundException e) { - builder.error(ImmutableErrorInfo.builder() - .logLevel(LogLevel.DEBUG) - .message( - "Could not determine GDS version, BuildInfoProperties is missing. " + - "This is likely due to not running GDS as a plugin, " + - "for example when running tests or using GDS as a Java module dependency." - ) - .reason(e) - .build() - ); - } catch (NoSuchMethodException | IllegalAccessException e) { - builder.error(ImmutableErrorInfo.builder() - .logLevel(LogLevel.WARN) - .message( - "Could not determine GDS version, the according methods on BuildInfoProperties could not be found.") - .reason(e) - .build() - ); - } catch (Throwable e) { - builder.error(ImmutableErrorInfo.builder() - .logLevel(LogLevel.WARN) - .message("Could not determine GDS version, the according methods on BuildInfoProperties failed.") - .reason(e) - .build() - ); - } - - return builder.gdsVersion("Unknown").build(); - } - private static final JavaInfo JAVA_INFO = loadJavaInfo(); private static JavaInfo loadJavaInfo() { diff --git a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java index b502f0e6154..c1ddd16bf8d 100644 --- a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java +++ b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java @@ -21,17 +21,14 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.core.cypher.CypherGraphStore; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -51,20 +48,12 @@ static void requireNeo4jVersion(Neo4jVersion version, Class self) { } } - CommandCreationContext inMemoryCommandCreationContext(); - void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, long sourceNodeId, int relTypeToken ); - StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, - TokenHolders tokenHolders, - CountsAccessor counts - ); - void createInMemoryDatabase( DatabaseManagementService dbms, String dbName, diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index ee819542b0b..6b42e165493 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; +import org.neo4j.common.DependencyResolver; import org.neo4j.configuration.Config; import org.neo4j.configuration.connectors.ConnectorPortRegister; import org.neo4j.dbms.api.DatabaseManagementService; @@ -52,6 +53,7 @@ import org.neo4j.internal.kernel.api.Read; import org.neo4j.internal.kernel.api.RelationshipScanCursor; import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.procs.FieldSignature; import org.neo4j.internal.kernel.api.procs.Neo4jTypes; import org.neo4j.internal.kernel.api.procs.ProcedureSignature; @@ -72,6 +74,8 @@ import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.query.TransactionalContext; import org.neo4j.kernel.impl.query.TransactionalContextFactory; @@ -318,7 +322,8 @@ public static ProcedureSignature procedureSignature( boolean caseInsensitive, boolean systemProcedure, boolean internal, - boolean allowExpiredCredentials + boolean allowExpiredCredentials, + boolean threadSafe ) { return IMPL.procedureSignature( name, @@ -333,7 +338,8 @@ public static ProcedureSignature procedureSignature( caseInsensitive, systemProcedure, internal, - allowExpiredCredentials + allowExpiredCredentials, + threadSafe ); } @@ -479,6 +485,19 @@ public static boolean isCompositeDatabase(GraphDatabaseService databaseService) return IMPL.isCompositeDatabase(databaseService); } + public static T lookupComponentProvider(Context ctx, Class component, boolean safe) throws + ProcedureException { + return IMPL.lookupComponentProvider(ctx, component, safe); + } + + public static GlobalProcedureRegistry globalProcedureRegistry(GlobalProcedures globalProcedures) { + return IMPL.globalProcedureRegistry(globalProcedures); + } + + public static DependencyResolver emptyDependencyResolver() { + return IMPL.emptyDependencyResolver(); + } + private Neo4jProxy() { throw new UnsupportedOperationException("No instances"); } diff --git a/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java index 4851dd1c07b..12964be986a 100644 --- a/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java +++ b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java @@ -21,18 +21,15 @@ import org.neo4j.common.Edition; import org.neo4j.configuration.Config; -import org.neo4j.counts.CountsAccessor; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.gds.core.cypher.CypherGraphStore; import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; import org.neo4j.io.layout.DatabaseLayout; -import org.neo4j.storageengine.api.CommandCreationContext; import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEntityCursor; import org.neo4j.storageengine.api.StoragePropertyCursor; -import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; @@ -45,10 +42,6 @@ public final class StorageEngineProxy { private StorageEngineProxy() {} - public static CommandCreationContext inMemoryCommandCreationContext() { - return IMPL.inMemoryCommandCreationContext(); - } - public static void initRelationshipTraversalCursorForRelType( StorageRelationshipTraversalCursor cursor, long sourceNodeId, @@ -57,12 +50,6 @@ public static void initRelationshipTraversalCursorForRelType( IMPL.initRelationshipTraversalCursorForRelType(cursor, sourceNodeId, relTypeToken); } - public static StorageReader inMemoryStorageReader( - CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts - ) { - return IMPL.inMemoryStorageReader(graphStore, tokenHolders, counts); - } - public static StorageEngine createInMemoryStorageEngine( DatabaseLayout databaseLayout, TokenHolders tokenHolders diff --git a/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java b/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java index 4f731fbd55c..43744b9adf5 100644 --- a/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java +++ b/concurrency-validation-api/src/main/java/org/neo4j/gds/concurrency/PoolSizesService.java @@ -19,6 +19,8 @@ */ package org.neo4j.gds.concurrency; +import java.util.Objects; + public final class PoolSizesService { private static PoolSizes instance = new OpenGdsPoolSizes(); @@ -31,6 +33,6 @@ public static void poolSizes(PoolSizes poolSizes) { } public static PoolSizes poolSizes() { - return instance; + return Objects.requireNonNull(instance); } } diff --git a/config-generator/build.gradle b/config-generator/build.gradle index 1faebbcd9c0..c86f22675ca 100644 --- a/config-generator/build.gradle +++ b/config-generator/build.gradle @@ -10,7 +10,7 @@ dependencies { annotationProcessor group: 'org.immutables', name: 'builder', version: ver.'immutables' annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' - implementation group: 'com.google.auto.service', name: 'auto-service', version: ver.'auto-service' + compileOnly group: 'com.google.auto.service', name: 'auto-service', version: ver.'auto-service' implementation project(':annotations') implementation group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' diff --git a/core-utils/src/main/java/org/neo4j/gds/utils/GdsFeatureToggles.java b/core-utils/src/main/java/org/neo4j/gds/utils/GdsFeatureToggles.java index a9017ae1de1..3bc4ac60f36 100644 --- a/core-utils/src/main/java/org/neo4j/gds/utils/GdsFeatureToggles.java +++ b/core-utils/src/main/java/org/neo4j/gds/utils/GdsFeatureToggles.java @@ -32,7 +32,6 @@ public enum GdsFeatureToggles { USE_PARALLEL_PROPERTY_VALUE_INDEX(false), USE_PARTITIONED_SCAN(false), USE_BIT_ID_MAP(true), - USE_SHARDED_ID_MAP(false), USE_UNCOMPRESSED_ADJACENCY_LIST(false), USE_PACKED_ADJACENCY_LIST(false), USE_REORDERED_ADJACENCY_LIST(false), diff --git a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipExporter.java b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipExporter.java index cb56c735589..42f9d1b62f5 100644 --- a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipExporter.java +++ b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipExporter.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.RelationshipIterator; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.partition.Partition; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -91,7 +91,7 @@ public static RelationshipExporterBuilder builder( this.propertyTranslator = propertyTranslator; this.terminationFlag = terminationFlag; this.progressTracker = progressTracker; - this.executorService = Pools.DEFAULT_SINGLE_THREAD_POOL; + this.executorService = ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL; } @Override diff --git a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterBuilder.java b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterBuilder.java index f1ebf2099ba..0eb0fae9475 100644 --- a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterBuilder.java +++ b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterBuilder.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.core.write; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.transaction.TransactionContext; public class NativeRelationshipPropertiesExporterBuilder extends RelationshipPropertiesExporterBuilder { @@ -37,7 +37,7 @@ public RelationshipPropertiesExporter build() { graphStore, propertyTranslator, progressTracker, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, terminationFlag ); } diff --git a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java index 45b14543428..53cc05e72fe 100644 --- a/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java +++ b/core-write/src/main/java/org/neo4j/gds/core/write/NativeRelationshipStreamExporter.java @@ -21,7 +21,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.transaction.TransactionContext; @@ -97,7 +97,7 @@ public long write(String relationshipType, List propertyKeys, List(bufferPool.poll()); diff --git a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java index 21eeaa48489..366743f2112 100644 --- a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java +++ b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodeLabelExporterTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.Neo4jGraph; @@ -58,7 +58,7 @@ void exportLabel(int concurrency) { TerminationFlag.RUNNING_TRUE, ProgressTracker.NULL_TRACKER, concurrency, - Pools.DEFAULT + DefaultPool.INSTANCE ); exporter.write("GeneratedLabel"); diff --git a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java index 0adb0f7c5d2..740e9d6d2d5 100644 --- a/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java +++ b/core-write/src/test/java/org/neo4j/gds/core/write/NativeNodePropertyExporterTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; @@ -149,7 +149,7 @@ void stopsExportingWhenTransactionHasBeenTerminated() { @Test void stopsParallelExportingWhenTransactionHasBeenTerminated() { - transactionTerminationTest(Pools.DEFAULT); + transactionTerminationTest(DefaultPool.INSTANCE); } @ParameterizedTest @@ -172,7 +172,7 @@ void progressLogging(boolean parallel) { .builder(TestSupport.fullAccessTransaction(db), graph, TerminationFlag.RUNNING_TRUE) .withProgressTracker(progressTracker); if (parallel) { - exporterBuilder = exporterBuilder.parallel(Pools.DEFAULT, writeConcurrency); + exporterBuilder = exporterBuilder.parallel(DefaultPool.INSTANCE, writeConcurrency); } var exporter = exporterBuilder.build(); diff --git a/core-write/src/test/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterTest.java b/core-write/src/test/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterTest.java index 2b98050c580..34899c4ab48 100644 --- a/core-write/src/test/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterTest.java +++ b/core-write/src/test/java/org/neo4j/gds/core/write/NativeRelationshipPropertiesExporterTest.java @@ -29,7 +29,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.values.storable.Values; @@ -79,7 +79,7 @@ void shouldWriteRelationshipsWithMultipleProperties() { graphStore, Values::doubleValue, ProgressTracker.NULL_TRACKER, - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, TerminationFlag.RUNNING_TRUE ); diff --git a/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java b/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java index 894cbf71644..95180032352 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphAdapter.java @@ -47,6 +47,11 @@ public Graph graph() { return graph; } + @Override + public String typeId() { + return graph.typeId(); + } + @Override public long relationshipCount() { return graph.relationshipCount(); diff --git a/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java b/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java index 020496d3ca4..c0bdda5442a 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphLoaderContext.java @@ -22,7 +22,7 @@ import org.immutables.value.Value; import org.neo4j.common.DependencyResolver; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -47,7 +47,7 @@ public interface GraphLoaderContext { @Value.Default default ExecutorService executor() { - return Pools.DEFAULT; + return DefaultPool.INSTANCE; } @Value.Default diff --git a/core/src/main/java/org/neo4j/gds/api/IdMap.java b/core/src/main/java/org/neo4j/gds/api/IdMap.java index 7ca084730ee..467bf4694fe 100644 --- a/core/src/main/java/org/neo4j/gds/api/IdMap.java +++ b/core/src/main/java/org/neo4j/gds/api/IdMap.java @@ -43,6 +43,16 @@ public interface IdMap extends PartialIdMap, NodeIterator, BatchNodeIterable { */ long NOT_FOUND = -1; + /** + * Used for IdMap implementations that do not require a type definition. + */ + String NO_TYPE = "unsupported"; + + /** + * A unique identifier for this type of IdMap. + */ + String typeId(); + /** * Map original nodeId to mapped nodeId * diff --git a/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java b/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java index 116d932fda7..dd43ab27ab8 100644 --- a/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/IdMapAdapter.java @@ -38,6 +38,11 @@ public IdMapAdapter(IdMap idMap) { this.idMap = idMap; } + @Override + public String typeId() { + return idMap.typeId(); + } + @Override public Collection batchIterables( long batchSize diff --git a/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java b/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java index b39614be5d3..103cc2a7e67 100644 --- a/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java +++ b/core/src/main/java/org/neo4j/gds/api/compress/AdjacencyCompressor.java @@ -49,7 +49,7 @@ public interface AdjacencyCompressor extends AutoCloseable { * @param degree The number of targets stored in `targets`. * @return the degree of the compressed adjacency list */ - int compress(long nodeId, long[] targets, long[][][] properties, int degree); + int compress(long nodeId, long[] targets, long[][] properties, int degree); /** * Closing this compressor will release some internal data structures, making them eligible for garbage collection. diff --git a/core/src/main/java/org/neo4j/gds/config/ConfigNodesValidations.java b/core/src/main/java/org/neo4j/gds/config/ConfigNodesValidations.java index 8174e228fa4..f5d00deaec3 100644 --- a/core/src/main/java/org/neo4j/gds/config/ConfigNodesValidations.java +++ b/core/src/main/java/org/neo4j/gds/config/ConfigNodesValidations.java @@ -40,6 +40,17 @@ static void validateNodes( String nodeDescription ) { if (!neoNodesToValidate.isEmpty()) { + + for (var neoNode : neoNodesToValidate) { + + if (neoNode < 0) { + throw new IllegalArgumentException(formatWithLocale( + "Negative node ids are not supported for the field `%s`", + nodeDescription + )); + } + } + var missingNodes = neoNodesToValidate .stream() .filter(targetNode -> labelFilteredGraphNotContainsNode( @@ -58,6 +69,7 @@ static void validateNodes( StringJoining.join(missingNodes) )); } + } } diff --git a/core/src/main/java/org/neo4j/gds/config/MutatePropertyConfig.java b/core/src/main/java/org/neo4j/gds/config/MutateNodePropertyConfig.java similarity index 97% rename from core/src/main/java/org/neo4j/gds/config/MutatePropertyConfig.java rename to core/src/main/java/org/neo4j/gds/config/MutateNodePropertyConfig.java index 3788a4e35c4..3be55dfd4ea 100644 --- a/core/src/main/java/org/neo4j/gds/config/MutatePropertyConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/MutateNodePropertyConfig.java @@ -31,7 +31,7 @@ import static org.neo4j.gds.core.StringIdentifierValidations.validateNoWhiteCharacter; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -public interface MutatePropertyConfig extends MutateConfig { +public interface MutateNodePropertyConfig extends MutateConfig { String MUTATE_PROPERTY_KEY = "mutateProperty"; diff --git a/core/src/main/java/org/neo4j/gds/config/MutateRelationshipPropertyConfig.java b/core/src/main/java/org/neo4j/gds/config/MutateRelationshipPropertyConfig.java new file mode 100644 index 00000000000..ec876ff497a --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/config/MutateRelationshipPropertyConfig.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.config; + +import org.jetbrains.annotations.Nullable; +import org.neo4j.gds.annotation.Configuration; + +import static org.neo4j.gds.core.StringIdentifierValidations.emptyToNull; +import static org.neo4j.gds.core.StringIdentifierValidations.validateNoWhiteCharacter; + +public interface MutateRelationshipPropertyConfig extends MutateConfig { + + String MUTATE_PROPERTY_KEY = "mutateProperty"; + + @Configuration.ConvertWith(method = "validateProperty") + @Configuration.Key(MUTATE_PROPERTY_KEY) + String mutateProperty(); + + static @Nullable String validateProperty(String input) { + return validateNoWhiteCharacter(emptyToNull(input), "mutateProperty"); + } + +} diff --git a/core/src/main/java/org/neo4j/gds/config/NodeConfig.java b/core/src/main/java/org/neo4j/gds/config/NodeConfig.java index 2d352d5acb3..f462963cd41 100644 --- a/core/src/main/java/org/neo4j/gds/config/NodeConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/NodeConfig.java @@ -29,7 +29,15 @@ static long parseNodeId(Object input, String field) { if (input instanceof Node) { return ((Node) input).getId(); } else if (input instanceof Number) { - return ((Number) input).longValue(); + var number = ((Number) input).longValue(); + if (number < 0) { + throw new IllegalArgumentException(formatWithLocale( + "Negative node ids are not supported for the field `%s`", + field, + input.getClass().getSimpleName() + )); + } + return number; } throw new IllegalArgumentException(formatWithLocale( diff --git a/core/src/main/java/org/neo4j/gds/config/SourceNodesConfig.java b/core/src/main/java/org/neo4j/gds/config/SourceNodesConfig.java index 1aeada122ff..389bb26482d 100644 --- a/core/src/main/java/org/neo4j/gds/config/SourceNodesConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/SourceNodesConfig.java @@ -45,7 +45,7 @@ default void validateSourceLabels( Collection selectedLabels, Collection selectedRelationshipTypes ) { - validateNodes(graphStore, sourceNodes(), selectedLabels, "Source"); + validateNodes(graphStore, sourceNodes(), selectedLabels, "sourceNodes"); } } diff --git a/core/src/main/java/org/neo4j/gds/config/TargetNodesConfig.java b/core/src/main/java/org/neo4j/gds/config/TargetNodesConfig.java index 34cd290af52..a8aabba7962 100644 --- a/core/src/main/java/org/neo4j/gds/config/TargetNodesConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/TargetNodesConfig.java @@ -51,6 +51,6 @@ default void validateTargetNodes( Collection selectedLabels, Collection selectedRelationshipTypes ) { - validateNodes(graphStore, targetNodes(), selectedLabels, "Target"); + validateNodes(graphStore, targetNodes(), selectedLabels, "targetNodes"); } } diff --git a/core/src/main/java/org/neo4j/gds/config/WriteConfig.java b/core/src/main/java/org/neo4j/gds/config/WriteConfig.java index 71f08f74c59..f316169aa59 100644 --- a/core/src/main/java/org/neo4j/gds/config/WriteConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/WriteConfig.java @@ -30,6 +30,7 @@ import org.neo4j.gds.core.CypherMapWrapper; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -61,6 +62,7 @@ default void validateWriteConcurrency() { * export builders. */ @Configuration.ConvertWith(method = "org.neo4j.gds.config.WriteConfig.ArrowConnectionInfo#parse") + @Configuration.ToMapValue(value="org.neo4j.gds.config.WriteConfig.ArrowConnectionInfo#toMap") Optional arrowConnectionInfo(); @Configuration.GraphStoreValidationCheck @@ -114,5 +116,13 @@ default boolean useEncryption() { input.getClass().getSimpleName() )); } + + static Map toMap(ArrowConnectionInfo info) { + Map map = new HashMap<>(); + map.put("hostname", info.hostname()); + map.put("port", info.port()); + map.put("useEncryption", info.useEncryption()); + return map; + } } } diff --git a/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java index 6540d712a74..a02da6eb0a0 100644 --- a/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/IdMapBehavior.java @@ -31,5 +31,20 @@ IdMapBuilder create( Optional nodeCount ); + /** + * Attempts to create an IdMapBuilder identified by the given id. + *

+ * If the id is not recognized, we fall back to the default behavior + * using {@link #create(int, Optional, Optional)}. + * + * @param id the id of the IdMapBuilder to create + */ + IdMapBuilder create( + String id, + int concurrency, + Optional maxOriginalId, + Optional nodeCount + ); + MemoryEstimation memoryEstimation(); } diff --git a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java index 9968f0c2f5c..88f73b80a41 100644 --- a/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java +++ b/core/src/main/java/org/neo4j/gds/core/OpenGdsIdMapBehavior.java @@ -22,9 +22,12 @@ import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.ArrayIdMapBuilder; import org.neo4j.gds.core.loading.GrowingArrayIdMapBuilder; +import org.neo4j.gds.core.loading.HighLimitIdMap; +import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.utils.mem.MemoryEstimation; +import java.util.Locale; import java.util.Optional; public class OpenGdsIdMapBehavior implements IdMapBehavior { @@ -40,6 +43,26 @@ public IdMapBuilder create( .orElseGet(GrowingArrayIdMapBuilder::of); } + @Override + public IdMapBuilder create(String id, int concurrency, Optional maxOriginalId, Optional nodeCount) { + var idLowerCase = id.toLowerCase(Locale.US); + if (idLowerCase.equals(ArrayIdMapBuilder.ID)) { + return create(concurrency, maxOriginalId, nodeCount); + } + if (HighLimitIdMap.isHighLimitIdMap(idLowerCase)) { + // We do not pass in the highest original id to the nested id map builder + // since initializing a HighLimitIdMap is typically a situation where the + // external ids may exceed the storage capabilities of the nested id map. + // Instead, we _know_ that the highest original id for the nested id map + // will be nodeCount - 1 as this is what the HighLimitIdMap guarantees. + var maxIntermediateId = nodeCount.map(nc -> nc - 1); + var innerBuilder = HighLimitIdMap.innerTypeId(idLowerCase) + .map(innerId -> create(innerId, concurrency, maxIntermediateId, nodeCount)) + .orElseGet(() -> create(concurrency, maxIntermediateId, nodeCount)); + return HighLimitIdMapBuilder.of(concurrency, innerBuilder); } + return create(concurrency, maxOriginalId, nodeCount); + } + @Override public MemoryEstimation memoryEstimation() { return ArrayIdMap.memoryEstimation(); diff --git a/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java b/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java index 8a11e8c50ea..a6e6b8d0bef 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/common/AdjacencyCompression.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.compression.common; import com.carrotsearch.hppc.sorting.IndirectSort; -import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.api.compress.AdjacencyCompressor; import org.neo4j.gds.api.compress.LongArrayBuffer; import org.neo4j.gds.core.Aggregation; @@ -39,12 +38,13 @@ public final class AdjacencyCompression { /** * Decompress the given {@code array} into the given {@code into}. - * After this, {@link org.neo4j.gds.api.compress.LongArrayBuffer#length} will reflect the number of decompressed values + * After this, {@link org.neo4j.gds.api.compress.LongArrayBuffer#length} will reflect the number of decompressed + * values * that are in the {@link org.neo4j.gds.api.compress.LongArrayBuffer#buffer}. */ public static void zigZagUncompressFrom( LongArrayBuffer into, - byte[][] targets, + byte[] targets, int compressedValues, int limit, AdjacencyCompressor.ValueMapper mapper @@ -56,7 +56,7 @@ public static void zigZagUncompressFrom( public static void zigZagUncompressFrom( long[] into, - byte[][] targets, + byte[] targets, int compressedValues, int limit, AdjacencyCompressor.ValueMapper mapper @@ -77,7 +77,7 @@ public static int applyDeltaEncoding(long[] data, int length, Aggregation aggreg // TODO: requires lots of additional memory ... inline indirect sort to make reuse of - to be created - buffers public static int applyDeltaEncoding( LongArrayBuffer data, - long[][][] weights, + long[][] weights, long[][] sortedWeights, Aggregation[] aggregations, boolean noAggregation @@ -96,7 +96,7 @@ public static int applyDeltaEncoding( public static int applyDeltaEncoding( long[] data, int length, - long[][][] unsortedWeights, + long[][] unsortedWeights, long[][] sortedWeights, Aggregation[] aggregations, boolean noAggregation @@ -233,21 +233,23 @@ public static void prefixSumDeltaEncodedValues(long[] values, int length) { * Applies delta encoding to the given {@code values}. * Weights are not encoded. * - * @param order Ordered indices into {@code values} and {@code weights} for consuming these in ascending value order. + * @param order Ordered indices into {@code values} and {@code weights} for consuming these in ascending + * value order. * @param values Relationships represented by target node ID. * @param outValues Sorted, delta-encoded and optionally aggregated relationships. * @param unsortedWeights Relationship properties by key, ordered by {@code order}. - * @param outWeights Sorted and optionally aggregated relationship properties. + * @param sortedWeights Sorted and optionally aggregated relationship properties. * @param length Number of relationships (degree of source node) to process. - * @param aggregations Aggregations to apply to parallel edges. One per relationship property key in {@code weights}. + * @param aggregations Aggregations to apply to parallel edges. One per relationship property key in + * {@code weights}. * @param noAggregation Is true iff all aggregations are NONE. */ private static int applyDelta( int[] order, long[] values, long[] outValues, - long[][][] unsortedWeights, - long[][] outWeights, + long[][] unsortedWeights, + long[][] sortedWeights, int length, Aggregation[] aggregations, boolean noAggregation @@ -256,41 +258,29 @@ private static int applyDelta( long value = values[firstSortIdx]; long delta; - int[] chunkLengths = new int[unsortedWeights[0].length]; - int totalChunkLength = 0; - for (int i = 0; i < unsortedWeights[0].length; i++) { - totalChunkLength += unsortedWeights[0][i].length; - chunkLengths[i] = totalChunkLength; - } - - var pageIndex = new MutableInt(); - var indexInPage = new MutableInt(); - findPosition(chunkLengths, firstSortIdx, pageIndex, indexInPage); - outValues[0] = values[firstSortIdx]; for (int i = 0; i < unsortedWeights.length; i++) { - outWeights[i][0] = unsortedWeights[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedWeights[i][0] = unsortedWeights[i][firstSortIdx]; } int in = 1, out = 1; for (; in < length; ++in) { final int sortIdx = order[in]; - findPosition(chunkLengths, sortIdx, pageIndex, indexInPage); delta = values[sortIdx] - value; value = values[sortIdx]; if (delta > 0L || noAggregation) { for (int i = 0; i < unsortedWeights.length; i++) { - outWeights[i][out] = unsortedWeights[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedWeights[i][out] = unsortedWeights[i][sortIdx]; } outValues[out++] = delta; } else { for (int i = 0; i < unsortedWeights.length; i++) { Aggregation aggregation = aggregations[i]; int existingIdx = out - 1; - long[] outWeight = outWeights[i]; + long[] outWeight = sortedWeights[i]; double existingWeight = Double.longBitsToDouble(outWeight[existingIdx]); - double newWeight = Double.longBitsToDouble(unsortedWeights[i][pageIndex.intValue()][indexInPage.intValue()]); + double newWeight = Double.longBitsToDouble(unsortedWeights[i][sortIdx]); newWeight = aggregation.merge(existingWeight, newWeight); outWeight[existingIdx] = Double.doubleToLongBits(newWeight); } @@ -299,18 +289,6 @@ private static int applyDelta( return out; } - public static void findPosition(int[] chunkLengths, int position, MutableInt pageIndex, MutableInt indexInPage) { - int chunkPosition = Arrays.binarySearch(chunkLengths, position); - if (chunkPosition >= 0) { - pageIndex.setValue(chunkPosition + 1); - indexInPage.setValue(0); - } else { - int index = -chunkPosition - 1; - pageIndex.setValue(index); - indexInPage.setValue(position - (index == 0 ? 0 : chunkLengths[index - 1])); - } - } - private AdjacencyCompression() { } } diff --git a/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java b/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java index d2dfa86d13c..374c294b8f3 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/common/ZigZagLongDecoding.java @@ -32,16 +32,24 @@ public long map(long value) { } } - public static int zigZagUncompress(byte[] chunk, int numberOfBytes, long[] out) { + public static int zigZagUncompress(byte[] compressedData, int length, long[] uncompressedData) { + return zigZagUncompress(compressedData, length, uncompressedData, Identity.INSTANCE); + } + + static int zigZagUncompress( + byte[] compressedData, + int length, + long[] uncompressedData, + AdjacencyCompressor.ValueMapper mapper + ) { long input, startValue = 0L, value = 0L; int into = 0, shift = 0, offset = 0; - - while (offset < numberOfBytes) { - input = chunk[offset++]; + while (offset < length) { + input = compressedData[offset++]; value += (input & 127L) << shift; if ((input & 128L) == 128L) { startValue += ((value >>> 1L) ^ -(value & 1L)); - out[into++] = startValue; + uncompressedData[into++] = mapper.map(startValue); value = 0L; shift = 0; } else { @@ -51,40 +59,6 @@ public static int zigZagUncompress(byte[] chunk, int numberOfBytes, long[] out) return into; } - public static int zigZagUncompress(byte[][] chunks, int numberOfBytes, long[] out) { - return zigZagUncompress(chunks, numberOfBytes, out, Identity.INSTANCE); - } - - public static int zigZagUncompress( - byte[][] chunks, - int numberOfBytes, - long[] out, - AdjacencyCompressor.ValueMapper mapper - ) { - int currentChunk = 0; - long input, startValue = 0L, value = 0L; - int into = 0, shift = 0; - - while (numberOfBytes > 0) { - var chunk = chunks[currentChunk++]; - var bytesToConsumeForChunk = Math.min(numberOfBytes, chunk.length); - for (int offset = 0; offset < bytesToConsumeForChunk; offset++) { - input = chunk[offset]; - value += (input & 127L) << shift; - if ((input & 128L) == 128L) { - startValue += ((value >>> 1L) ^ -(value & 1L)); - out[into++] = mapper.map(startValue); - value = 0L; - shift = 0; - } else { - shift += 7; - } - } - numberOfBytes -= bytesToConsumeForChunk; - } - return into; - } - private ZigZagLongDecoding() { throw new UnsupportedOperationException("No instances"); } diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java index ec6ea1307e6..a6a84c8ec0f 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacker.java @@ -165,6 +165,7 @@ private static long runPacking( Address address = slice.slice(); long ptr = address.address() + slice.offset(); + long initialPtr = ptr; // write header UnsafeUtil.copyMemory(header, BYTE_ARRAY_BASE_OFFSET, null, ptr, headerSize); @@ -177,6 +178,9 @@ private static long runPacking( in += AdjacencyPacking.BLOCK_SIZE; } + if (ptr > initialPtr + allocationSize) + throw new AssertionError("Written more bytes than allocated. ptr=" + ptr + ", initialPtr=" + initialPtr + ", allocationSize=" + allocationSize); + return adjacencyOffset; } @@ -330,7 +334,7 @@ private static long preparePackingWithPackedTail( if (hasTail) { int bits = bitsNeeded(values, offset, tailLength); memoryTracker.recordHeaderBits(bits); - bytes += BitUtil.ceilDiv((long) bits * tailLength, Long.BYTES); + bytes += bytesNeeded(bits, tailLength); header[blockIdx] = (byte) bits; } @@ -370,6 +374,7 @@ private static long runPackingWithPackedTail( Address address = slice.slice(); long ptr = address.address() + slice.offset(); + long initialPtr = ptr; // write header UnsafeUtil.copyMemory(header, BYTE_ARRAY_BASE_OFFSET, null, ptr, headerSize); @@ -389,9 +394,13 @@ private static long runPackingWithPackedTail( // tail packing if (hasTail) { byte bits = header[header.length - 1]; - AdjacencyPacking.loopPack(bits, values, in, tailLength, ptr); + ptr = AdjacencyPacking.loopPack(bits, values, in, tailLength, ptr); } + if (ptr > initialPtr + allocationSize) + throw new AssertionError("Written more bytes than allocated. ptr=" + ptr + ", initialPtr=" + initialPtr + ", allocationSize=" + allocationSize); + + return adjacencyOffset; } @@ -406,4 +415,8 @@ private static int bitsNeeded(long[] values, int offset, int length) { private static int bytesNeeded(int bits) { return BitUtil.ceilDiv(AdjacencyPacking.BLOCK_SIZE * bits, Byte.SIZE); } + + private static int bytesNeeded(int bits, int length) { + return BitUtil.ceilDiv(length * bits, Byte.SIZE); + } } diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java index fb162fd1f16..3ef7c3d9961 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/AdjacencyPacking.java @@ -10884,8 +10884,10 @@ private static long packLoop1(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -10909,8 +10911,10 @@ private static long packLoop2(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -10934,8 +10938,10 @@ private static long packLoop3(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -10959,8 +10965,10 @@ private static long packLoop4(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -10984,8 +10992,10 @@ private static long packLoop5(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11009,8 +11019,10 @@ private static long packLoop6(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11034,8 +11046,10 @@ private static long packLoop7(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11059,8 +11073,10 @@ private static long packLoop8(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11084,8 +11100,10 @@ private static long packLoop9(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11109,8 +11127,10 @@ private static long packLoop10(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11134,8 +11154,10 @@ private static long packLoop11(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11159,8 +11181,10 @@ private static long packLoop12(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11184,8 +11208,10 @@ private static long packLoop13(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11209,8 +11235,10 @@ private static long packLoop14(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11234,8 +11262,10 @@ private static long packLoop15(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11259,8 +11289,10 @@ private static long packLoop16(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11284,8 +11316,10 @@ private static long packLoop17(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11309,8 +11343,10 @@ private static long packLoop18(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11334,8 +11370,10 @@ private static long packLoop19(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11359,8 +11397,10 @@ private static long packLoop20(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11384,8 +11424,10 @@ private static long packLoop21(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11409,8 +11451,10 @@ private static long packLoop22(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11434,8 +11478,10 @@ private static long packLoop23(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11459,8 +11505,10 @@ private static long packLoop24(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11484,8 +11532,10 @@ private static long packLoop25(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11509,8 +11559,10 @@ private static long packLoop26(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11534,8 +11586,10 @@ private static long packLoop27(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11559,8 +11613,10 @@ private static long packLoop28(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11584,8 +11640,10 @@ private static long packLoop29(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11609,8 +11667,10 @@ private static long packLoop30(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11634,8 +11694,10 @@ private static long packLoop31(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11659,8 +11721,10 @@ private static long packLoop32(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11684,8 +11748,10 @@ private static long packLoop33(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11709,8 +11775,10 @@ private static long packLoop34(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11734,8 +11802,10 @@ private static long packLoop35(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11759,8 +11829,10 @@ private static long packLoop36(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11784,8 +11856,10 @@ private static long packLoop37(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11809,8 +11883,10 @@ private static long packLoop38(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11834,8 +11910,10 @@ private static long packLoop39(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11859,8 +11937,10 @@ private static long packLoop40(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11884,8 +11964,10 @@ private static long packLoop41(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11909,8 +11991,10 @@ private static long packLoop42(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11934,8 +12018,10 @@ private static long packLoop43(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11959,8 +12045,10 @@ private static long packLoop44(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -11984,8 +12072,10 @@ private static long packLoop45(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12009,8 +12099,10 @@ private static long packLoop46(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12034,8 +12126,10 @@ private static long packLoop47(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12059,8 +12153,10 @@ private static long packLoop48(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12084,8 +12180,10 @@ private static long packLoop49(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12109,8 +12207,10 @@ private static long packLoop50(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12134,8 +12234,10 @@ private static long packLoop51(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12159,8 +12261,10 @@ private static long packLoop52(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12184,8 +12288,10 @@ private static long packLoop53(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12209,8 +12315,10 @@ private static long packLoop54(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12234,8 +12342,10 @@ private static long packLoop55(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12259,8 +12369,10 @@ private static long packLoop56(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12284,8 +12396,10 @@ private static long packLoop57(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12309,8 +12423,10 @@ private static long packLoop58(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12334,8 +12450,10 @@ private static long packLoop59(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12359,8 +12477,10 @@ private static long packLoop60(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12384,8 +12504,10 @@ private static long packLoop61(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12409,8 +12531,10 @@ private static long packLoop62(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } @@ -12434,8 +12558,10 @@ private static long packLoop63(long[] values, int valuesStart, int valuesLength, shift -= 64; } } - UnsafeUtil.putLong(packedPtr, word); - packedPtr += 8; + if (shift != 0) { + UnsafeUtil.putLong(packedPtr, word); + packedPtr += 8; + } return packedPtr; } diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java index cef5cda7bdf..4f222e11376 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/PackedCompressor.java @@ -50,7 +50,8 @@ public static AdjacencyCompressorFactory factory( boolean noAggregation, MemoryTracker memoryTracker ) { - AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings.numberOfMappings()]; + AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings + .numberOfMappings()]; Arrays.setAll(propertyBuilders, i -> adjacencyListBuilderFactory.newAdjacencyPropertiesBuilder(memoryTracker)); return new Factory( @@ -171,7 +172,7 @@ private PackedCompressor( } @Override - public int compress(long nodeId, long[] targets, long[][][] properties, int degree) { + public int compress(long nodeId, long[] targets, long[][] properties, int degree) { if (properties != null) { return packWithProperties( nodeId, @@ -191,7 +192,7 @@ public int compress(long nodeId, long[] targets, long[][][] properties, int degr private int packWithProperties( long nodeId, long[] targets, - long[][][] unsortedProperties, + long[][] unsortedProperties, int degree ) { long[][] sortedProperties = new long[unsortedProperties.length][degree]; diff --git a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs index 6073e3aebd0..870b6a50343 100755 --- a/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs +++ b/core/src/main/java/org/neo4j/gds/core/compression/packed/gen_packing.rs @@ -571,7 +571,7 @@ fn single_pack(bits: u32, offset: u32) -> Inst { } mod java { - use std::io::Write; + use std::{collections::HashSet, io::Write}; use super::*; use gen_java::*; @@ -804,19 +804,27 @@ mod java { }); } Inst::PackLoopRemainder => { - let mut if_body = vec![]; - - if_body.push(Stmt::Expr(Expr::Call(Call::new( + let mut then = vec![]; + then.push(Stmt::Expr(Expr::Call(Call::new( Expr::Ident("UnsafeUtil"), "putLong", vec![Arg::new(Expr::Ident(PW)), Arg::new(Expr::Ident(WORD))], )))); - if_body.push(Stmt::assign_op( + then.push(Stmt::assign_op( Expr::Ident(PW), Expr::Literal(8), BinOp::Add, )); - statements.extend(if_body); + + if method.bits == 64 { + statements.extend(then); + } else { + statements.push(Stmt::If { + cond: Expr::bin(Expr::Ident(SHIFT), BinOp::Neq, Expr::Literal(0)), + then: Box::new(Stmt::Block(then)), + ells: None, + }); + } } Inst::UnpackLoop => { // PIN[offset + i + OFF] @@ -1192,7 +1200,7 @@ mod java { } } - fn gen_class(class: Class) -> ClassDef { + fn gen_class(class: Class) -> (ClassDef, Vec) { fn gen_assert(bs: u32) -> Stmt { Stmt::Assert { assertion: Expr::bin(Expr::Ident(BITS), BinOp::Lte, Expr::Literal(bs)), @@ -1204,6 +1212,8 @@ mod java { } } + let mut imports = HashSet::new(); + let mut members = vec![ Member::Method(MethodDef { documentation: None, @@ -1234,6 +1244,8 @@ mod java { ]; if !class.packers.is_empty() { + imports.insert("org.neo4j.internal.unsafe.UnsafeUtil"); + members.extend([ Member::Method(MethodDef { documentation: None, @@ -1292,6 +1304,8 @@ mod java { } if !class.unpackers.is_empty() { + imports.insert("org.neo4j.internal.unsafe.UnsafeUtil"); + members.extend([ Member::Method(MethodDef { documentation: None, @@ -1350,6 +1364,8 @@ mod java { } if !class.loop_packers.is_empty() { + imports.insert("org.neo4j.internal.unsafe.UnsafeUtil"); + members.extend([ Member::Method(MethodDef { documentation: None, @@ -1413,6 +1429,9 @@ mod java { } if !class.loop_unpackers.is_empty() { + imports.insert("org.neo4j.internal.unsafe.UnsafeUtil"); + imports.insert("org.neo4j.gds.mem.BitUtil"); + members.extend([ Member::Method(MethodDef { documentation: None, @@ -1488,18 +1507,23 @@ mod java { let doc = class.documentation.join("\n"); - ClassDef { + let class = ClassDef { documentation: Some(doc), annotations: Vec::new(), modifiers: "public final", typ: "class", name: class.name, members, - } + }; + + let mut imports = imports.into_iter().map(String::from).collect::>(); + imports.sort_unstable(); + + (class, imports) } fn gen_file(file: File) -> FileDef { - let class = gen_class(file.class); + let (class, imports) = gen_class(file.class); let mut file = FileDef::new( r#" Copyright (c) "Neo4j" @@ -1521,10 +1545,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . "#, file.package, - vec![ - "org.neo4j.internal.unsafe.UnsafeUtil".into(), - "org.neo4j.gds.mem.BitUtil".into(), - ], + imports, class, ); diff --git a/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java index badfd097cb5..a589f4dff52 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/uncompressed/RawCompressor.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.compression.uncompressed; import com.carrotsearch.hppc.sorting.IndirectSort; -import org.apache.commons.lang3.mutable.MutableInt; import org.jetbrains.annotations.Nullable; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.api.AdjacencyList; @@ -40,8 +39,6 @@ import java.util.Arrays; import java.util.function.LongSupplier; -import static org.neo4j.gds.core.compression.common.AdjacencyCompression.findPosition; - public final class RawCompressor implements AdjacencyCompressor { public static AdjacencyCompressorFactory factory( @@ -52,8 +49,10 @@ public static AdjacencyCompressorFactory factory( boolean noAggregation, MemoryTracker memoryTracker ) { - @SuppressWarnings("unchecked") - AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings.numberOfMappings()]; + @SuppressWarnings( + "unchecked" + ) AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings + .numberOfMappings()]; Arrays.setAll(propertyBuilders, i -> adjacencyListBuilderFactory.newAdjacencyPropertiesBuilder(memoryTracker)); return new Factory( @@ -158,7 +157,7 @@ private RawCompressor( } @Override - public int compress(long nodeId, long[] targets, long[][][] properties, int degree) { + public int compress(long nodeId, long[] targets, long[][] properties, int degree) { if (properties != null) { return withProperties( nodeId, @@ -168,7 +167,9 @@ public int compress(long nodeId, long[] targets, long[][][] properties, int degr ); } else { return withoutProperties( - nodeId, targets, degree + nodeId, + targets, + degree ); } } @@ -222,32 +223,21 @@ private int aggregate(long[] values, int length, Aggregation aggregation) { private int aggregateWithProperties( long[] values, int length, - long[][][] unsortedProperties, + long[][] unsortedProperties, long[][] sortedProperties, Aggregation[] aggregations ) { int[] order = IndirectSort.mergesort(0, length, new AscendingLongComparator(values)); + long[] outValues = new long[length]; + int firstSortIdx = order[0]; long value = values[firstSortIdx]; long delta; - - int[] chunkLengths = new int[unsortedProperties[0].length]; - int totalChunkLength = 0; - for (int i = 0; i < unsortedProperties[0].length; i++) { - totalChunkLength += unsortedProperties[0][i].length; - chunkLengths[i] = totalChunkLength; - } - - var pageIndex = new MutableInt(); - var indexInPage = new MutableInt(); - findPosition(chunkLengths, firstSortIdx, pageIndex, indexInPage); - - long[] outValues = new long[length]; outValues[0] = value; for (int i = 0; i < unsortedProperties.length; i++) { - sortedProperties[i][0] = unsortedProperties[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedProperties[i][0] = unsortedProperties[i][firstSortIdx]; } int in = 1, out = 1; @@ -255,10 +245,9 @@ private int aggregateWithProperties( if (this.noAggregation) { for (; in < length; ++in) { int sortIdx = order[in]; - findPosition(chunkLengths, sortIdx, pageIndex, indexInPage); for (int i = 0; i < unsortedProperties.length; i++) { - sortedProperties[i][out] = unsortedProperties[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedProperties[i][out] = unsortedProperties[i][sortIdx]; } outValues[out++] = values[sortIdx]; @@ -266,13 +255,12 @@ private int aggregateWithProperties( } else { for (; in < length; ++in) { int sortIdx = order[in]; - findPosition(chunkLengths, sortIdx, pageIndex, indexInPage); delta = values[sortIdx] - value; value = values[sortIdx]; if (delta > 0L) { for (int i = 0; i < unsortedProperties.length; i++) { - sortedProperties[i][out] = unsortedProperties[i][pageIndex.intValue()][indexInPage.intValue()]; + sortedProperties[i][out] = unsortedProperties[i][sortIdx]; } outValues[out++] = value; } else { @@ -281,7 +269,7 @@ private int aggregateWithProperties( int existingIdx = out - 1; long[] outProperty = sortedProperties[i]; double existingProperty = Double.longBitsToDouble(outProperty[existingIdx]); - double newProperty = Double.longBitsToDouble(unsortedProperties[i][pageIndex.intValue()][indexInPage.intValue()]); + double newProperty = Double.longBitsToDouble(unsortedProperties[i][sortIdx]); newProperty = aggregation.merge(existingProperty, newProperty); outProperty[existingIdx] = Double.doubleToLongBits(newProperty); } @@ -290,6 +278,9 @@ private int aggregateWithProperties( } System.arraycopy(outValues, 0, values, 0, out); + for (int i = 0; i < sortedProperties.length; i++) { + System.arraycopy(sortedProperties[i], 0, unsortedProperties[i], 0, out); + } return out; @@ -298,7 +289,7 @@ private int aggregateWithProperties( private int withProperties( long nodeId, long[] targets, - long[][][] unsortedProperties, + long[][] unsortedProperties, int degree ) { long[][] sortedProperties = new long[unsortedProperties.length][degree]; diff --git a/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java b/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java index 19547efe4cf..310e3cb3d33 100644 --- a/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java +++ b/core/src/main/java/org/neo4j/gds/core/compression/varlong/DeltaVarLongCompressor.java @@ -49,8 +49,10 @@ public static AdjacencyCompressorFactory factory( boolean noAggregation, MemoryTracker memoryTracker ) { - @SuppressWarnings("unchecked") - AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings.numberOfMappings()]; + @SuppressWarnings( + "unchecked" + ) AdjacencyListBuilder[] propertyBuilders = new AdjacencyListBuilder[propertyMappings + .numberOfMappings()]; Arrays.setAll(propertyBuilders, i -> adjacencyListBuilderFactory.newAdjacencyPropertiesBuilder(memoryTracker)); return new Factory( @@ -157,7 +159,7 @@ private DeltaVarLongCompressor( } @Override - public int compress(long nodeId, long[] targets, long[][][] properties, int degree) { + public int compress(long nodeId, long[] targets, long[][] properties, int degree) { if (properties != null) { return applyVariableDeltaEncodingWithProperties( nodeId, @@ -210,7 +212,7 @@ private int applyVariableDeltaEncodingWithoutProperties(long nodeId, long[] targ private int applyVariableDeltaEncodingWithProperties( long nodeId, long[] targets, - long[][][] unsortedProperties, + long[][] unsortedProperties, int degree ) { // buffer contains uncompressed, unsorted target list diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java b/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java new file mode 100644 index 00000000000..1ba55b09b2e --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/DefaultPool.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.concurrency; + +import org.neo4j.gds.concurrency.PoolSizes; +import org.neo4j.gds.concurrency.PoolSizesService; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +public final class DefaultPool { + + public static final ExecutorService INSTANCE = createDefaultPool(PoolSizesService.poolSizes()); + + private static ExecutorService createDefaultPool(PoolSizes poolSizes) { + return new ThreadPoolExecutor( + poolSizes.corePoolSize(), + poolSizes.maxPoolSize(), + 30L, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(poolSizes.corePoolSize() * 50), + ExecutorServiceUtil.DEFAULT_THREAD_FACTORY, + new ExecutorServiceUtil.CallerBlocksPolicy() + ); + } + + private DefaultPool() {} +} diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java b/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServiceUtil.java similarity index 83% rename from core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java rename to core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServiceUtil.java index a08703dfb77..111de7433e5 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/Pools.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/ExecutorServiceUtil.java @@ -19,10 +19,8 @@ */ package org.neo4j.gds.core.concurrency; -import org.neo4j.gds.concurrency.PoolSizesService; import org.neo4j.internal.helpers.NamedThreadFactory; -import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -35,31 +33,17 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -public final class Pools { +public final class ExecutorServiceUtil { private static final String THREAD_NAME_PREFIX = "gds"; public static final ThreadFactory DEFAULT_THREAD_FACTORY = NamedThreadFactory.daemon(THREAD_NAME_PREFIX); - public static final ExecutorService DEFAULT = createDefaultPool(); public static final ExecutorService DEFAULT_SINGLE_THREAD_POOL = createSingleThreadPool("algo"); - private Pools() { + private ExecutorServiceUtil() { throw new UnsupportedOperationException(); } - public static ExecutorService createDefaultPool() { - var poolSizes = PoolSizesService.poolSizes(); - return new ThreadPoolExecutor( - poolSizes.corePoolSize(), - poolSizes.maxPoolSize(), - 30L, - TimeUnit.SECONDS, - new ArrayBlockingQueue<>(poolSizes.corePoolSize() * 50), - DEFAULT_THREAD_FACTORY, - new CallerBlocksPolicy() - ); - } - public static ExecutorService createSingleThreadPool(String threadPrefix) { return Executors.newSingleThreadExecutor(NamedThreadFactory.daemon(threadPrefix)); } @@ -103,7 +87,7 @@ public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { private static final ForkJoinPool.ForkJoinWorkerThreadFactory FJ_WORKER_THREAD_FACTORY = pool -> { var worker = ForkJoinPool.defaultForkJoinWorkerThreadFactory.newThread(pool); - worker.setName(Pools.THREAD_NAME_PREFIX + "-forkjoin-" + worker.getPoolIndex()); + worker.setName(ExecutorServiceUtil.THREAD_NAME_PREFIX + "-forkjoin-" + worker.getPoolIndex()); return worker; }; } diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/ParallelUtil.java b/core/src/main/java/org/neo4j/gds/core/concurrency/ParallelUtil.java index 2f57d486a68..9e0bf799e82 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/ParallelUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/ParallelUtil.java @@ -70,7 +70,7 @@ private ParallelUtil() {} * The concurrency value is assumed to already be validated towards the edition limitation. */ public static , R> R parallelStream(T data, int concurrency, Function fn) { - ForkJoinPool pool = Pools.createForkJoinPool(concurrency); + ForkJoinPool pool = ExecutorServiceUtil.createForkJoinPool(concurrency); try { return pool.submit(() -> fn.apply(data.parallel())).get(); } catch (InterruptedException e) { diff --git a/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java b/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java index 5d4506fc42c..0928ce08ec6 100644 --- a/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java +++ b/core/src/main/java/org/neo4j/gds/core/concurrency/RunWithConcurrency.java @@ -186,11 +186,11 @@ default boolean mayInterruptIfRunning() { * If {@link #forceUsageOfExecutor()} is {@code true} however, an {@link java.lang.IllegalArgumentException} * is thrown when this object is constructed. *

- * The default executor is {@link org.neo4j.gds.core.concurrency.Pools#DEFAULT}. + * The default executor is {@link DefaultPool#INSTANCE}. */ @Value.Default default @Nullable ExecutorService executor() { - return Pools.DEFAULT; + return DefaultPool.INSTANCE; } /** diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java index 652fc35a8c9..b55d835f9cc 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyCursor.java @@ -29,7 +29,6 @@ import java.util.PriorityQueue; public class CompositeAdjacencyCursor implements AdjacencyCursor { - private final PriorityQueue cursorQueue; private final List cursors; @@ -52,6 +51,11 @@ public List cursors() { return cursors; } + void reinitializeCursors() { + cursorQueue.clear(); + initializeQueue(); + + } @Override public int size() { int sum = 0; diff --git a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java index cd8e3eadeac..9f084715544 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/CompositeAdjacencyList.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.AdjacencyCursor; import org.neo4j.gds.api.AdjacencyList; -import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.FilteredIdMap; import org.neo4j.gds.api.ImmutableMemoryInfo; import java.util.ArrayList; @@ -61,12 +61,9 @@ static CompositeAdjacencyList of(List adjacencyLists) { ); } - static CompositeAdjacencyList withFilteredIdMap(List adjacencyLists, IdMap filteredIdMap) { - assert filteredIdMap instanceof NodeFilteredGraph; - var adjacencyCursorWrapperFactory = (AdjacencyCursorWrapperFactory) cursor -> new NodeFilteredAdjacencyCursor( - cursor, - filteredIdMap - ); + static CompositeAdjacencyList withFilteredIdMap(List adjacencyLists, FilteredIdMap filteredIdMap) { + var adjacencyCursorWrapperFactory = (AdjacencyCursorWrapperFactory) cursor -> new NodeFilteredAdjacencyCursor(cursor, filteredIdMap); + var compositeAdjacencyCursorFactory = (CompositeAdjacencyCursorFactory) cursors -> { List wrappedCursors = cursors .stream() @@ -128,6 +125,7 @@ public CompositeAdjacencyCursor adjacencyCursor(@Nullable AdjacencyCursor reuse, if (reuse instanceof CompositeAdjacencyCursor) { var compositeReuse = (CompositeAdjacencyCursor) reuse; var iter = compositeReuse.cursors().listIterator(); + while (iter.hasNext()) { var index = iter.nextIndex(); var cursor = iter.next(); @@ -136,6 +134,10 @@ public CompositeAdjacencyCursor adjacencyCursor(@Nullable AdjacencyCursor reuse, iter.set(adjacencyCursorWrapperFactory.create(newCursor)); } } + //we must update all the cursors all the time because old cursor might be further ahead instead of start + //then, object equality can be then the same, but in reality not so much + compositeReuse.reinitializeCursors(); + return compositeReuse; } return adjacencyCursor(node, fallbackValue); diff --git a/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java b/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java index 31333dd98c5..e158334f686 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/HugeGraph.java @@ -393,6 +393,11 @@ public long toMappedNodeId(long originalNodeId) { return idMap.toMappedNodeId(originalNodeId); } + @Override + public String typeId() { + return idMap.typeId(); + } + @Override public long toOriginalNodeId(long mappedNodeId) { return idMap.toOriginalNodeId(mappedNodeId); diff --git a/core/src/main/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursor.java b/core/src/main/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursor.java index eb0f9613b5a..591141e255d 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursor.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursor.java @@ -22,18 +22,18 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.neo4j.gds.api.AdjacencyCursor; -import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.FilteredIdMap; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public class NodeFilteredAdjacencyCursor implements AdjacencyCursor { private final AdjacencyCursor innerCursor; - private final IdMap idMap; + private final FilteredIdMap idMap; private long nextLongValue; - NodeFilteredAdjacencyCursor(AdjacencyCursor innerCursor, IdMap idMap) { + NodeFilteredAdjacencyCursor(AdjacencyCursor innerCursor, FilteredIdMap idMap) { this.innerCursor = innerCursor; this.idMap = idMap; @@ -57,11 +57,11 @@ public int size() { public boolean hasNextVLong() { if (innerCursor.hasNextVLong()) { var innerNextLong = innerCursor.peekVLong(); - if (!idMap.containsOriginalId(innerNextLong)) { + if (!idMap.containsRootNodeId(innerNextLong)) { innerCursor.nextVLong(); return hasNextVLong(); } - this.nextLongValue = idMap.toMappedNodeId(innerNextLong); + this.nextLongValue = idMap.toFilteredNodeId(innerNextLong); return true; } return false; diff --git a/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java b/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java index 9855f19eb32..49f02c6d114 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/UnionGraph.java @@ -150,6 +150,11 @@ public long toMappedNodeId(long originalNodeId) { return first.toMappedNodeId(originalNodeId); } + @Override + public String typeId() { + return first.typeId(); + } + @Override public long toOriginalNodeId(long mappedNodeId) { return first.toOriginalNodeId(mappedNodeId); @@ -333,10 +338,10 @@ public CompositeAdjacencyList relationshipTopology() { .flatMap(Collection::stream) .map(Topology::adjacencyList) .collect(Collectors.toList()); - if (isNodeFilteredGraph()) { - return CompositeAdjacencyList.withFilteredIdMap(adjacencies, first); - } - return CompositeAdjacencyList.of(adjacencies); + + return first.asNodeFilteredGraph() + .map(graph -> CompositeAdjacencyList.withFilteredIdMap(adjacencies, graph)) + .orElseGet(() -> CompositeAdjacencyList.of(adjacencies)); } @Override diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java index cb7011084a6..c8d597bae5a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java @@ -89,6 +89,11 @@ public long toMappedNodeId(long originalNodeId) { return originalToInternalIds.get(originalNodeId); } + @Override + public String typeId() { + return ArrayIdMapBuilder.ID; + } + @Override public long toOriginalNodeId(long mappedNodeId) { return internalToOriginalIds.get(mappedNodeId); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java index 6b1e526a407..1e68da0b906 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java @@ -28,6 +28,8 @@ public final class ArrayIdMapBuilder implements IdMapBuilder { + public static final String ID = "array"; + private final HugeLongArray array; private final long capacity; private final AtomicLong allocationIndex; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java index d414d5938e5..b1723d17310 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java @@ -23,8 +23,8 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.collections.HugeSparseLongArray; import org.neo4j.gds.collections.cursor.HugeCursor; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.construction.NodesBuilder; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -75,7 +75,7 @@ static HugeSparseLongArray buildSparseIdMap( ParallelUtil.readParallel( concurrency, nodeCount, - Pools.DEFAULT, + DefaultPool.INSTANCE, (start, end) -> addNodes(graphIds, idMapBuilder, start, end) ); return idMapBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java b/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java index d9e56dcae17..eadb65bb24b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java @@ -20,10 +20,10 @@ package org.neo4j.gds.core.loading; import org.neo4j.gds.collections.DrainingIterator; -import org.neo4j.gds.collections.HugeSparseByteArrayArrayList; +import org.neo4j.gds.collections.HugeSparseByteArrayList; import org.neo4j.gds.collections.HugeSparseCollections; import org.neo4j.gds.collections.HugeSparseIntList; -import org.neo4j.gds.collections.HugeSparseLongArrayArrayList; +import org.neo4j.gds.collections.HugeSparseLongArrayList; import org.neo4j.gds.collections.HugeSparseLongList; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -40,20 +40,19 @@ import static org.neo4j.gds.core.compression.common.VarLongEncoding.zigZag; import static org.neo4j.gds.core.loading.AdjacencyPreAggregation.IGNORE_VALUE; import static org.neo4j.gds.mem.BitUtil.ceilDiv; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public final class ChunkedAdjacencyLists { - private static final byte[][] EMPTY_BYTES_BYTES = new byte[0][]; - private static final long[][] EMPTY_PROPERTIES = new long[0][]; + private static final byte[] EMPTY_BYTES = new byte[0]; + private static final long[] EMPTY_PROPERTIES = new long[0]; - private final HugeSparseByteArrayArrayList targetLists; - private final HugeSparseLongArrayArrayList[] properties; + private final HugeSparseByteArrayList targetLists; + private final HugeSparseLongArrayList[] properties; private final HugeSparseIntList positions; private final HugeSparseLongList lastValues; private final HugeSparseIntList lengths; - private byte[] compressBuffer = new byte[1 << 20]; // 1MiB - public static MemoryEstimation memoryEstimation(long avgDegree, long nodeCount, int propertyCount) { // Best case scenario: // Difference between node identifiers in each adjacency list is 1. @@ -73,7 +72,10 @@ public static MemoryEstimation memoryEstimation(long avgDegree, long nodeCount, .fixed("positions", HugeSparseCollections.estimateInt(nodeCount, nodeCount)) .fixed("lengths", HugeSparseCollections.estimateInt(nodeCount, nodeCount)) .fixed("lastValues", HugeSparseCollections.estimateLong(nodeCount, nodeCount)) - .fixed("properties", HugeSparseCollections.estimateLongArray(nodeCount, nodeCount, (int) avgDegree).times(propertyCount)) + .fixed( + "properties", + HugeSparseCollections.estimateLongArray(nodeCount, nodeCount, (int) avgDegree).times(propertyCount) + ) .build(); } @@ -89,15 +91,15 @@ public static ChunkedAdjacencyLists of(int numberOfProperties, long initialCapac } private ChunkedAdjacencyLists(int numberOfProperties, long initialCapacity) { - this.targetLists = HugeSparseByteArrayArrayList.of(EMPTY_BYTES_BYTES, initialCapacity); + this.targetLists = HugeSparseByteArrayList.of(EMPTY_BYTES, initialCapacity); this.positions = HugeSparseIntList.of(0, initialCapacity); this.lastValues = HugeSparseLongList.of(0, initialCapacity); this.lengths = HugeSparseIntList.of(0, initialCapacity); if (numberOfProperties > 0) { - this.properties = new HugeSparseLongArrayArrayList[numberOfProperties]; - Arrays.setAll(this.properties, i -> HugeSparseLongArrayArrayList.of(EMPTY_PROPERTIES, initialCapacity)); + this.properties = new HugeSparseLongArrayList[numberOfProperties]; + Arrays.setAll(this.properties, i -> HugeSparseLongArrayList.of(EMPTY_PROPERTIES, initialCapacity)); } else { this.properties = null; } @@ -128,14 +130,11 @@ public void add(long index, long[] targets, int start, int end, int valuesToAdd) } var position = positions.get(index); - if (requiredBytes > this.compressBuffer.length) { - this.compressBuffer = new byte[BitUtil.nextHighestPowerOfTwo(requiredBytes)]; - } - encodeVLongs(targets, start, end, this.compressBuffer, 0); + var compressedTargets = ensureCompressedTargetsCapacity(index, position, requiredBytes); - copyTargetData(index, position, this.compressBuffer, requiredBytes, this.targetLists); + var newPosition = encodeVLongs(targets, start, end, compressedTargets, position); - positions.set(index, position + requiredBytes); + positions.set(index, newPosition); this.lastValues.set(index, currentLastValue); this.lengths.addTo(index, valuesToAdd); @@ -153,192 +152,81 @@ public void add(long index, long[] targets, int start, int end, int valuesToAdd) public void add(long index, long[] targets, long[][] allProperties, int start, int end, int targetsToAdd) { // write properties for (int i = 0; i < allProperties.length; i++) { - var length = lengths.get(index); - copyPropertiesData( - index, - length, - allProperties[i], - start, - end, - targets, - targetsToAdd, - this.properties[i] - ); + addProperties(index, targets, allProperties[i], start, end, i, targetsToAdd); } // write values add(index, targets, start, end, targetsToAdd); } - static final int[] NEXT_CHUNK_LENGTH = { - 32, - 32, - 64, - 64, - 128, - 128, - 256, - 256, - 512, - 512, - 1024, - 1024, - 2048, - 4096, - 8192, - 16384, - 32768, - 65536, - 131072, - 262144, - 524288, - 1048576, - 2097152, - 4194304, - }; - - // This method and the one after that ("copyPropertiesData") are the same except for the underlying primitive array type. - // Unfortunately, writing this method to be generic over the primitive array type requires us to use methods - // that are either reflection based or not optimizable by C2, and it shows on benchmarks. - // Any changes made here should also be done in "copyPropertiesData". - private static void copyTargetData( + private void addProperties( long index, - int targetPos, - byte[] buffer, - int bufferLength, - HugeSparseByteArrayArrayList dataLists + long[] targets, + long[] properties, + int start, + int end, + int propertyIndex, + int propertiesToAdd ) { - byte[][] compressedData = dataLists.get(index); + var length = lengths.get(index); - // last chunk, write pos = posInLastChunk - int posInLastChunk = targetPos; - for (int targetPage = 0; targetPage < compressedData.length - 1; targetPage++) { - byte[] compressedTarget = compressedData[targetPage]; - posInLastChunk -= compressedTarget.length; - } + var currentProperties = ensurePropertyCapacity(index, length, propertiesToAdd, propertyIndex); - // can we fit everything in the last chunk? - if (compressedData.length > 0) { - byte[] lastChunk = compressedData[compressedData.length - 1]; - if (lastChunk.length >= posInLastChunk + bufferLength) { - // fits in last chunk - System.arraycopy(buffer, 0, lastChunk, posInLastChunk, bufferLength); - return; + if (propertiesToAdd == end - start) { + System.arraycopy(properties, start, currentProperties, length, propertiesToAdd); + } else { + var writePos = length; + for (int i = 0; i < (end - start); i++) { + if (targets[start + i] != IGNORE_VALUE) { + currentProperties[writePos++] = properties[start + i]; + } } } + } - // how many bytes do we need to write into the last chunk - int fitsInLastChunk = compressedData.length == 0 - ? 0 - : compressedData[compressedData.length - 1].length - posInLastChunk; - - int newChunkLengthAtLeast = bufferLength - fitsInLastChunk; - - // the next chunk size grows slowly depending on the number of chunks we already have - int nextChunkLevel = Math.min(compressedData.length, NEXT_CHUNK_LENGTH.length - 1); - int nextChunkLength = NEXT_CHUNK_LENGTH[nextChunkLevel]; - - // avoid splitting the buffer into too many chunks - int newChunkLength = Math.max(newChunkLengthAtLeast, nextChunkLength); - - // copy buffer into the last chunk and the new chunk - byte[] newChunk = new byte[newChunkLength]; - if (fitsInLastChunk > 0) { - System.arraycopy( - buffer, - 0, - compressedData[compressedData.length - 1], - posInLastChunk, - fitsInLastChunk + private byte[] ensureCompressedTargetsCapacity(long index, int pos, int required) { + int targetLength = pos + required; + var compressedTargets = targetLists.get(index); + + if (targetLength < 0) { + throw new IllegalArgumentException( + formatWithLocale( + "Encountered numeric overflow in internal buffer. Was at position %d and needed to grow by %d.", + pos, + required + ) ); + } else if (compressedTargets.length <= targetLength) { + int newLength = BitUtil.nextHighestPowerOfTwo(targetLength); +// int newLength = ArrayUtil.oversize(pos + required, Byte.BYTES); + compressedTargets = Arrays.copyOf(compressedTargets, newLength); + this.targetLists.set(index, compressedTargets); } - System.arraycopy(buffer, fitsInLastChunk, newChunk, 0, bufferLength - fitsInLastChunk); - // add new chunk to the targets list - compressedData = Arrays.copyOf(compressedData, compressedData.length + 1); - compressedData[compressedData.length - 1] = newChunk; - dataLists.set(index, compressedData); + return compressedTargets; } - // This method and the one before that ("copyTargetData") are the same except for the underlying primitive array type. - // Unfortunately, writing this method to be generic over the primitive array type requires us to use methods - // that are either reflection based or not optimizable by C2, and it shows on benchmarks. - // Any changes made here should also be done in "copyTargetData". - private static void copyPropertiesData( - long index, - int targetPos, - long[] buffer, - int bufferOffset, - int bufferEnd, - long[] targets, - int bufferLength, - HugeSparseLongArrayArrayList dataLists - ) { - if (bufferEnd - bufferOffset != bufferLength) { - // compact the property buffer and ignore properties from ignored targets - var writeIndex = bufferOffset; - for (int i = bufferOffset; i < bufferEnd; i++) { - if (targets[i] != IGNORE_VALUE) { - if (writeIndex != i) { - buffer[writeIndex] = buffer[i]; - } - writeIndex++; - } - } + private long[] ensurePropertyCapacity(long index, int pos, int required, int propertyIndex) { + int targetLength = pos + required; - assert bufferLength == writeIndex - bufferOffset; - } - - long[][] compressedData = dataLists.get(index); - - // last chunk, write pos = posInLastChunk - int posInLastChunk = targetPos; - for (int targetPage = 0; targetPage < compressedData.length - 1; targetPage++) { - long[] compressedTarget = compressedData[targetPage]; - posInLastChunk -= compressedTarget.length; - } - - // can we fit everything in the last chunk? - if (compressedData.length > 0) { - long[] lastChunk = compressedData[compressedData.length - 1]; - if (lastChunk.length >= posInLastChunk + bufferLength) { - // fits in last chunk - System.arraycopy(buffer, bufferOffset, lastChunk, posInLastChunk, bufferLength); - return; - } - } + var currentProperties = this.properties[propertyIndex].get(index); - // how many bytes do we need to write into the last chunk - int fitsInLastChunk = compressedData.length == 0 - ? 0 - : compressedData[compressedData.length - 1].length - posInLastChunk; - - int newChunkLengthAtLeast = bufferLength - fitsInLastChunk; - - // the next chunk size grows slowly depending on the number of chunks we already have - int nextChunkLevel = Math.min(compressedData.length, NEXT_CHUNK_LENGTH.length - 1); - int nextChunkLength = NEXT_CHUNK_LENGTH[nextChunkLevel] / 2; - - // avoid splitting the buffer into too many chunks - int newChunkLength = Math.max(newChunkLengthAtLeast, nextChunkLength); - - // copy buffer into the last chunk and the new chunk - long[] newChunk = new long[newChunkLength]; - if (fitsInLastChunk > 0) { - System.arraycopy( - buffer, - bufferOffset, - compressedData[compressedData.length - 1], - posInLastChunk, - fitsInLastChunk + if (targetLength < 0) { + throw new IllegalArgumentException( + formatWithLocale( + "Encountered numeric overflow in internal buffer. Was at position %d and needed to grow by %d.", + pos, + required + ) ); + } else if (currentProperties.length <= pos + required) { +// int newLength = ArrayUtil.oversize(pos + required, Long.BYTES); + int newLength = BitUtil.nextHighestPowerOfTwo(pos + required); + currentProperties = Arrays.copyOf(currentProperties, newLength); + this.properties[propertyIndex].set(index, currentProperties); } - System.arraycopy(buffer, fitsInLastChunk + bufferOffset, newChunk, 0, bufferLength - fitsInLastChunk); - // add new chunk to the targets list - compressedData = Arrays.copyOf(compressedData, compressedData.length + 1); - compressedData[compressedData.length - 1] = newChunk; - dataLists.set(index, compressedData); + return currentProperties; } public long capacity() { @@ -356,30 +244,30 @@ public void consume(Consumer consumer) { public interface Consumer { void accept( long sourceId, - byte[][] targets, - long[][][] properties, + byte[] targets, + long[][] properties, int compressedByteSize, int numberOfCompressedTargets ); } private static class CompositeDrainingIterator { - private final DrainingIterator targetListIterator; - private final DrainingIterator.DrainingBatch targetListBatch; + private final DrainingIterator targetListIterator; + private final DrainingIterator.DrainingBatch targetListBatch; private final DrainingIterator positionsListIterator; private final DrainingIterator.DrainingBatch positionsListBatch; private final DrainingIterator lastValuesListIterator; private final DrainingIterator.DrainingBatch lastValuesListBatch; private final DrainingIterator lengthsListIterator; private final DrainingIterator.DrainingBatch lengthsListBatch; - private final List> propertyIterators; - private final List> propertyBatches; + private final List> propertyIterators; + private final List> propertyBatches; - private final long[][][] propertiesBuffer; + private final long[][] propertiesBuffer; CompositeDrainingIterator( - HugeSparseByteArrayArrayList targets, - HugeSparseLongArrayArrayList[] properties, + HugeSparseByteArrayList targets, + HugeSparseLongArrayList[] properties, HugeSparseIntList positions, HugeSparseLongList lastValues, HugeSparseIntList lengths @@ -399,13 +287,13 @@ private static class CompositeDrainingIterator { propertiesBuffer = null; } else { this.propertyIterators = Arrays.stream(properties) - .map(HugeSparseLongArrayArrayList::drainingIterator) + .map(HugeSparseLongArrayList::drainingIterator) .collect(Collectors.toList()); this.propertyBatches = this.propertyIterators .stream() .map(DrainingIterator::drainingBatch) .collect(Collectors.toList()); - propertiesBuffer = new long[properties.length][][]; + propertiesBuffer = new long[properties.length][]; } } @@ -426,21 +314,19 @@ public void consume(Consumer consumer) { for (int indexInPage = 0; indexInPage < targetsPage.length; indexInPage++) { var targets = targetsPage[indexInPage]; - if (targets == EMPTY_BYTES_BYTES) { + if (targets == EMPTY_BYTES) { continue; } - // make targets eligible for GC - targetsPage[indexInPage] = null; - var position = positionsPage[indexInPage]; var length = lengthsPage[indexInPage]; for (int propertyIndex = 0; propertyIndex < propertyBatches.size(); propertyIndex++) { - long[][][] page = propertyBatches.get(propertyIndex).page; - + long[][] page = propertyBatches.get(propertyIndex).page; propertiesBuffer[propertyIndex] = page[indexInPage]; // make properties eligible for GC page[indexInPage] = null; } + // make targets eligible for GC + targetsPage[indexInPage] = null; consumer.accept(offset + indexInPage, targets, propertiesBuffer, position, length); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java index dd1b91827f6..3755610730c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CountingCypherRecordLoader.java @@ -21,8 +21,10 @@ import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.config.GraphProjectFromCypherConfig; +import org.neo4j.graphdb.security.AuthorizationViolationException; import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import java.util.Locale; import java.util.Set; class CountingCypherRecordLoader extends CypherRecordLoader { @@ -45,6 +47,19 @@ class CountingCypherRecordLoader extends CypherRecordLoader { BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { var subscriber = new ResultCountingSubscriber(); CypherLoadingUtils.consume(runLoadingQuery(tx, subscriber)); + + subscriber.error().ifPresent(e -> { + if (e instanceof AuthorizationViolationException) { + throw new IllegalArgumentException(String.format( + Locale.US, + "Query must be read only. Query: [%s]", + this.loadQuery + )); + } + + throw new RuntimeException(e); + }); + return new BatchLoadResult(subscriber.rows(), -1L); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java index 2f9f603b587..5d11e1a105f 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.loading; +import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.transaction.TransactionContext; @@ -55,6 +56,8 @@ public EstimationResult runEstimationQuery(String query, Collection rese var propertyColumns = new ArrayList<>(result.columns()); propertyColumns.removeAll(reservedColumns); + propertyColumns.forEach(PropertyMapping::validatePropertyKey); + return ImmutableEstimationResult.of(estimatedRows.longValue(), propertyColumns.size()); } }); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java index b52313c777b..672b1e2ca04 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java @@ -62,7 +62,7 @@ String toLowerCase() { final GraphLoaderContext loadingContext; private final long recordCount; - private final String loadQuery; + protected final String loadQuery; private final QueryExecutionEngine executionEngine; private final TransactionalContextFactory contextFactory; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java index 137fa500448..f061c6aa2bf 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/FilteredLabeledIdMap.java @@ -38,6 +38,11 @@ public FilteredLabeledIdMap(IdMap originalToRootIdMap, LabeledIdMap rootToFilter this.rootToFilteredIdMap = rootToFilteredIdMap; } + @Override + public String typeId() { + return originalToRootIdMap.typeId(); + } + @Override public OptionalLong rootNodeCount() { return originalToRootIdMap.rootNodeCount(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java index a05b4e00196..7cecb160cc9 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMap.java @@ -37,6 +37,11 @@ public HighLimitIdMap(ShardedLongLongMap intermediateIdMap, IdMap internalIdMap) this.highToLowIdSpace = intermediateIdMap; } + @Override + public String typeId() { + return HighLimitIdMapBuilder.ID + "-" + super.typeId(); + } + @Override public long toOriginalNodeId(long mappedNodeId) { return highToLowIdSpace.toOriginalNodeId(super.toOriginalNodeId(mappedNodeId)); @@ -71,6 +76,19 @@ public Optional withFilteredLabels(Collection nodeLabe .map(filteredIdMap -> new FilteredHighLimitIdMap(this.highToLowIdSpace, filteredIdMap)); } + public static boolean isHighLimitIdMap(String typeId) { + return typeId.startsWith(HighLimitIdMapBuilder.ID); + } + + public static Optional innerTypeId(String typeId) { + var separatorIndex = typeId.indexOf('-'); + if (isHighLimitIdMap(typeId) && separatorIndex > 0 && separatorIndex < typeId.length() - 1) { + var substring = typeId.substring(separatorIndex + 1); + return substring.equals(HighLimitIdMapBuilder.ID) ? Optional.empty() : Optional.of(substring); + } + return Optional.empty(); + } + static final class FilteredHighLimitIdMap extends HighLimitIdMap implements FilteredIdMap { private final FilteredIdMap filteredIdMap; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java new file mode 100644 index 00000000000..0d101f6d2ee --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilder.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; +import org.neo4j.gds.utils.CloseableThreadLocal; + +public final class HighLimitIdMapBuilder implements IdMapBuilder { + + public static final String ID = "highlimit"; + + private final ShardedLongLongMap.BatchedBuilder originalToIntermediateMapping; + private final IdMapBuilder intermediateToInternalMapping; + + private final CloseableThreadLocal bulkAdders; + + public static HighLimitIdMapBuilder of(int concurrency, IdMapBuilder internalIdMapBuilder) { + return new HighLimitIdMapBuilder(concurrency, internalIdMapBuilder); + } + + private HighLimitIdMapBuilder(int concurrency, IdMapBuilder internalIdMapBuilder) { + // We use a builder that overrides the node ids in the input batch with the + // generated intermediate node ids. This is necessary for downstream label + // and property processing. + this.originalToIntermediateMapping = ShardedLongLongMap.batchedBuilder(concurrency, true); + this.intermediateToInternalMapping = internalIdMapBuilder; + this.bulkAdders = CloseableThreadLocal.withInitial(this::newBulkAdder); + } + + @Override + public IdMapAllocator allocate(int batchLength) { + var batch = this.originalToIntermediateMapping.prepareBatch(batchLength); + var internalAllocator = this.intermediateToInternalMapping.allocate(batchLength); + var bulkAdder = this.bulkAdders.get(); + bulkAdder.reset(batchLength, internalAllocator, batch); + return bulkAdder; + } + + @Override + public IdMap build(LabelInformation.Builder labelInformationBuilder, long highestNodeId, int concurrency) { + var intermediateIdMap = this.originalToIntermediateMapping.build(); + var highestIntermediateId = intermediateIdMap.size() - 1; + var internalIdMap = this.intermediateToInternalMapping.build(labelInformationBuilder, highestIntermediateId, concurrency); + + return new HighLimitIdMap(intermediateIdMap, internalIdMap); + } + + private BulkAdder newBulkAdder() { + return new BulkAdder(); + } + + private static final class BulkAdder implements IdMapAllocator { + + private IdMapAllocator intermediateAllocator; + private IdMapAllocator internalAllocator; + private int batchLength = 0; + + private void reset(int batchLength, IdMapAllocator internalAllocator, IdMapAllocator intermediateAllocator) { + this.batchLength = batchLength; + this.internalAllocator = internalAllocator; + this.intermediateAllocator = intermediateAllocator; + } + + @Override + public int allocatedSize() { + return this.batchLength; + } + + @Override + public void insert(long[] nodeIds) { + this.intermediateAllocator.insert(nodeIds); + this.internalAllocator.insert(nodeIds); + } + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index e3d3e442184..f73203b4780 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -30,7 +30,6 @@ import org.neo4j.gds.core.loading.construction.PropertyValues; import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; -import java.util.Locale; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicBoolean; @@ -40,7 +39,12 @@ public final class LazyIdMapBuilder implements PartialIdMap { private final NodesBuilder nodesBuilder; - public LazyIdMapBuilder(int concurrency, boolean hasLabelInformation, boolean hasProperties, PropertyState propertyState) { + public LazyIdMapBuilder( + int concurrency, + boolean hasLabelInformation, + boolean hasProperties, + PropertyState propertyState + ) { this.intermediateIdMapBuilder = ShardedLongLongMap.builder(concurrency); this.nodesBuilder = GraphFactory.initNodesBuilder() .concurrency(concurrency) @@ -57,7 +61,7 @@ public void prepareForFlush() { } public long addNode(long nodeId, NodeLabelToken nodeLabels) { - checkPositiveId(nodeId); + LoadingExceptions.checkPositiveId(nodeId); long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); @@ -71,19 +75,6 @@ public long addNode(long nodeId, NodeLabelToken nodeLabels) { return intermediateId; } - /** - * GDS has the general assumption of non-negative original node ids. - */ - private static void checkPositiveId(long nodeId) { - if (nodeId < 0) { - throw new IllegalArgumentException(String.format( - Locale.US, - "GDS expects node ids to be positive. But got a negative id of `%d`.", - nodeId - )); - } - } - public long addNodeWithProperties( long nodeId, PropertyValues properties, @@ -91,7 +82,7 @@ public long addNodeWithProperties( ) { long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); - checkPositiveId(nodeId); + LoadingExceptions.checkPositiveId(nodeId); // deduplication if (intermediateId < 0) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LoadingExceptions.java b/core/src/main/java/org/neo4j/gds/core/loading/LoadingExceptions.java index e24aabf139d..48b1d0e56d0 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LoadingExceptions.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LoadingExceptions.java @@ -32,13 +32,28 @@ public static void validateSourceNodeIsLoaded(long mappedId, long neoId) { validateNodeIsLoaded(mappedId, neoId, "source"); } + /** + * GDS has the general assumption of non-negative original node ids. + */ + public static void checkPositiveId(long nodeId) { + if (nodeId < 0) { + throw new IllegalArgumentException( + String.format( + Locale.US, + "GDS expects node ids to be positive. But got a negative id of `%d`.", + nodeId + ) + ); + } + } + private static void validateNodeIsLoaded(long mappedId, long neoId, String side) { if (mappedId == -1) { throw new IllegalArgumentException( String.format( Locale.US, "Failed to load a relationship because its %s-node with id %s is not part of the node query or projection. " + - "To ignore the relationship, set the configuration parameter `validateRelationships` to false.", + "To ignore the relationship, set the configuration parameter `validateRelationships` to false.", side, neoId ) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NodeSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/NodeSubscriber.java index 99e31f19d08..67970c753bb 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NodeSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NodeSubscriber.java @@ -21,10 +21,9 @@ import org.neo4j.gds.core.loading.construction.NodeLabelTokens; import org.neo4j.gds.core.loading.construction.NodesBuilder; +import org.neo4j.gds.core.utils.ErrorCachingQuerySubscriber; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.graphdb.QueryStatistics; -import org.neo4j.kernel.impl.query.QueryExecutionKernelException; -import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; import org.neo4j.values.SequenceValue; import org.neo4j.values.storable.NumberValue; @@ -34,12 +33,11 @@ import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.Set; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -class NodeSubscriber implements QuerySubscriber { +class NodeSubscriber extends ErrorCachingQuerySubscriber { private static final String ID_COLUMN = "id"; private static final int UNINITIALIZED = -1; @@ -62,8 +60,6 @@ class NodeSubscriber implements QuerySubscriber { private int labelOffset = UNINITIALIZED; private String[] fieldNames; - private Optional error = Optional.empty(); - public NodeSubscriber( ProgressTracker progressTracker ) { @@ -83,11 +79,6 @@ void initialize(String[] fieldNames, NodesBuilder nodesBuilder) { } } - Optional error() - { - return error; - } - long rows() { return rows; } @@ -150,17 +141,6 @@ public void onRecordCompleted() { progressTracker.logProgress(); } - @Override - public void onError(Throwable throwable) { - if (throwable instanceof RuntimeException) { - this.error = Optional.of((RuntimeException) throwable); - } else if (throwable instanceof QueryExecutionKernelException) { - this.error = Optional.of(((QueryExecutionKernelException) throwable).asUserException()); - } else { - this.error = Optional.of(new RuntimeException(throwable)); - } - } - @Override public void onResultCompleted(QueryStatistics statistics) { } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NodesBatchBuffer.java b/core/src/main/java/org/neo4j/gds/core/loading/NodesBatchBuffer.java index da86e7fbc7c..9f027630bb3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NodesBatchBuffer.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NodesBatchBuffer.java @@ -120,10 +120,6 @@ public boolean offer(final NodeReference record) { } public void add(long nodeId, PropertyReference propertyReference, long[] labels) { - if (nodeId >= this.highestPossibleNodeCount) { - return; - } - int len = length++; buffer[len] = nodeId; if (properties != null) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java index 996c9cea897..883bb69e4ba 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java @@ -23,22 +23,20 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; +import org.neo4j.gds.core.utils.ErrorCachingQuerySubscriber; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.graphdb.QueryStatistics; -import org.neo4j.kernel.impl.query.QueryExecutionKernelException; -import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; import org.neo4j.values.storable.NumberValue; import org.neo4j.values.storable.TextValue; -import java.util.Optional; import java.util.Set; import static org.neo4j.gds.RelationshipType.ALL_RELATIONSHIPS; import static org.neo4j.gds.core.loading.LoadingExceptions.validateSourceNodeIsLoaded; import static org.neo4j.gds.core.loading.LoadingExceptions.validateTargetNodeIsLoaded; -class RelationshipSubscriber implements QuerySubscriber { +class RelationshipSubscriber extends ErrorCachingQuerySubscriber { private static final String SOURCE_COLUMN = "source"; private static final String TARGET_COLUMN = "target"; @@ -67,8 +65,6 @@ class RelationshipSubscriber implements QuerySubscriber { private RelationshipsBuilder allRelationshipsBuilder; - private Optional error = Optional.empty(); - RelationshipSubscriber( IdMap idMap, CypherRelationshipLoader.Context loaderContext, @@ -109,10 +105,6 @@ void initialize(String[] fieldNames, ObjectDoubleMap propertyDefaultValu this.propertyValueBuffer = new double[propertyCount]; } - Optional error() { - return error; - } - public long rows() { return rows; } @@ -189,16 +181,6 @@ public void onRecordCompleted() { progressTracker.logProgress(); } - @Override - public void onError(Throwable throwable) { - if (throwable instanceof RuntimeException) { - this.error = Optional.of((RuntimeException) throwable); - } else if (throwable instanceof QueryExecutionKernelException) { - this.error = Optional.of(((QueryExecutionKernelException) throwable).asUserException()); - } else { - this.error = Optional.of(new RuntimeException(throwable)); - } - } @Override public void onResultCompleted(QueryStatistics statistics) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java index 5df91e8a670..12b2feb59ae 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ResultCountingSubscriber.java @@ -19,11 +19,11 @@ */ package org.neo4j.gds.core.loading; +import org.neo4j.gds.core.utils.ErrorCachingQuerySubscriber; import org.neo4j.graphdb.QueryStatistics; -import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; -class ResultCountingSubscriber implements QuerySubscriber { +class ResultCountingSubscriber extends ErrorCachingQuerySubscriber { private long rows = 0; public long rows() { @@ -51,11 +51,6 @@ public void onRecordCompleted() { } - @Override - public void onError(Throwable throwable) { - - } - @Override public void onResultCompleted(QueryStatistics statistics) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 93d3a3863d6..5bc86c2f772 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -199,7 +199,12 @@ public Nodes build() { importPropertiesFromIndex(idMap, nodeProperties); } - return Nodes.of(idMap, this.propertyMappingsByLabel, nodeProperties, PropertyState.PERSISTENT); + return Nodes.of( + idMap, + this.propertyMappingsByLabel, + nodeProperties, + PropertyState.PERSISTENT + ); } private void importPropertiesFromIndex( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 467d9c86a6d..3ba8858e931 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -40,9 +40,10 @@ import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.huge.HugeGraphBuilder; +import org.neo4j.gds.core.loading.HighLimitIdMap; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.loading.ImmutableImportMetaData; import org.neo4j.gds.core.loading.ImportSizing; @@ -85,7 +86,8 @@ static NodesBuilder nodesBuilder( Optional hasProperties, Optional deduplicateIds, Optional concurrency, - Optional propertyState + Optional propertyState, + Optional idMapBuilderType ) { boolean labelInformation = nodeSchema .map(schema -> !(schema.availableLabels().isEmpty() && schema.containsOnlyAllNodesLabel())) @@ -97,35 +99,60 @@ static NodesBuilder nodesBuilder( ? Optional.of(maxOriginalId) : Optional.empty(); + var idMapType = idMapBuilderType.orElse(IdMap.NO_TYPE); var idMapBuilder = idMapBehavior.create( + idMapType, threadCount, maybeMaxOriginalId, nodeCount ); boolean deduplicate = deduplicateIds.orElse(true); + long maxIntermediateId = maxOriginalId; + + if (HighLimitIdMap.isHighLimitIdMap(idMapType)) { + // If the requested id map is high limit, we need to make sure that + // internal data structures are sized accordingly. Using the highest + // original id will potentially fail due to size limitations. + if (nodeCount.isPresent()) { + maxIntermediateId = nodeCount.get() - 1; + } else { + throw new IllegalArgumentException("Cannot use high limit id map without node count."); + } + if (deduplicate) { + // We internally use HABS for deduplication, which is being initialized + // with max original id. This is fine for all id maps except high limit, + // where original ids can exceed the supported HABS range. + throw new IllegalArgumentException("Cannot use high limit id map with deduplication."); + } + } - return nodeSchema.map(schema -> fromSchema( - maxOriginalId, - idMapBuilder, - threadCount, - schema, - labelInformation, - deduplicate - )).orElseGet(() -> new NodesBuilder( - maxOriginalId, - threadCount, - NodesBuilderContext.lazy(threadCount), - idMapBuilder, - labelInformation, - hasProperties.orElse(false), - deduplicate, - __ -> propertyState.orElse(PropertyState.PERSISTENT) - )); + return nodeSchema.isPresent() + ? fromSchema( + maxOriginalId, + maxIntermediateId, + idMapBuilder, + threadCount, + nodeSchema.get(), + labelInformation, + deduplicate + ) + : new NodesBuilder( + maxOriginalId, + maxIntermediateId, + threadCount, + NodesBuilderContext.lazy(threadCount), + idMapBuilder, + labelInformation, + hasProperties.orElse(false), + deduplicate, + __ -> propertyState.orElse(PropertyState.PERSISTENT) + ); } private static NodesBuilder fromSchema( long maxOriginalId, + long maxIntermediateId, IdMapBuilder idMapBuilder, int concurrency, NodeSchema nodeSchema, @@ -134,6 +161,7 @@ private static NodesBuilder fromSchema( ) { return new NodesBuilder( maxOriginalId, + maxIntermediateId, concurrency, NodesBuilderContext.fixed(nodeSchema, concurrency), idMapBuilder, @@ -267,7 +295,7 @@ static RelationshipsBuilder relationshipsBuilder( .isMultiGraph(isMultiGraph) .loadRelationshipProperty(loadRelationshipProperties) .direction(Direction.fromOrientation(actualOrientation)) - .executorService(executorService.orElse(Pools.DEFAULT)) + .executorService(executorService.orElse(DefaultPool.INSTANCE)) .concurrency(finalConcurrency); if (indexInverse.orElse(false)) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 9dad02bc635..1988bfe0933 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -78,6 +78,7 @@ public final class NodesBuilder { NodesBuilder( long maxOriginalId, + long maxIntermediateId, int concurrency, NodesBuilderContext nodesBuilderContext, IdMapBuilder idMapBuilder, @@ -93,7 +94,7 @@ public final class NodesBuilder { this.propertyStates = propertyStates; this.labelInformationBuilder = !hasLabelInformation ? LabelInformationBuilders.allNodes() - : LabelInformationBuilders.multiLabelWithCapacity(maxOriginalId + 1); + : LabelInformationBuilders.multiLabelWithCapacity(maxIntermediateId + 1); this.importedNodes = new LongAdder(); this.nodeImporter = new NodeImporterBuilder() diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java index 1bb7ac3fbbb..d7db8d37e18 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java @@ -24,8 +24,8 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.DoubleArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseDoubleArrayArray; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -91,7 +91,7 @@ public DoubleArrayNodePropertyValues build(long size, PartialIdMap idMap, long h } } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java index e5b45a1e484..f7b112e8a90 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java @@ -24,8 +24,8 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.DoubleNodePropertyValues; import org.neo4j.gds.collections.HugeSparseDoubleArray; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -116,7 +116,7 @@ public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highes } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java index 86633552e96..8b5563b678e 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java @@ -24,8 +24,8 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.FloatArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseFloatArrayArray; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -90,7 +90,7 @@ public FloatArrayNodePropertyValues build(long size, PartialIdMap idMap, long hi } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); return new FloatArrayStoreNodePropertyValues(propertyValues, size); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java index 6c3616b641a..ddfbb31ef22 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java @@ -24,8 +24,8 @@ import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArrayArray; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -94,7 +94,7 @@ public LongArrayNodePropertyValues build(long size, PartialIdMap idMap, long hig } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java index e743d718d13..7e3752cbcbf 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java @@ -25,8 +25,8 @@ import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArray; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.utils.Neo4jValueConversion; import org.neo4j.values.storable.Value; @@ -124,7 +124,7 @@ public NodePropertyValues build(long size, PartialIdMap idMap, long highestOrigi } }).collect(Collectors.toList()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var propertyValues = propertiesByMappedIdsBuilder.build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java index df891258ee3..e6f4e819e43 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java @@ -24,6 +24,7 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.collections.HugeSparseCollections; +import org.neo4j.gds.core.loading.HighLimitIdMap; import org.neo4j.gds.core.loading.ValueConverter; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -90,7 +91,13 @@ public NodePropertyValues build(IdMap idMap) { } } - return innerBuilder.get().build(idMap.nodeCount(), idMap, idMap.highestOriginalId()); + // For HighLimitIdMap, we need to use the rootIdMap to resolve intermediate + // node ids correctly. The rootIdMap in that case is the mapping between + // intermediate and mapped node ids. The imported property values are associated + // with the intermediate node ids. + var actualIdMap = (idMap instanceof HighLimitIdMap) ? idMap.rootIdMap() : idMap; + + return innerBuilder.get().build(idMap.nodeCount(), actualIdMap, idMap.highestOriginalId()); } // This is synchronized as we want to prevent the creation of multiple InnerNodePropertiesBuilders of which only once survives. diff --git a/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java b/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java new file mode 100644 index 00000000000..d8076f86805 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/utils/ErrorCachingQuerySubscriber.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.utils; + +import org.jetbrains.annotations.Nullable; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.kernel.impl.query.QuerySubscriber; +import org.neo4j.values.AnyValue; + +import java.util.Optional; + +public abstract class ErrorCachingQuerySubscriber implements QuerySubscriber { + + @Nullable + private RuntimeException error; + + public Optional error() { + return Optional.ofNullable(error); + } + + public void onError(Throwable throwable) { + if (throwable instanceof RuntimeException) { + this.error = (RuntimeException) throwable; + } else if (throwable instanceof QueryExecutionKernelException) { + this.error = ((QueryExecutionKernelException) throwable).asUserException(); + } else { + this.error = new RuntimeException(throwable); + } + } + + public static final class DoNothingSubscriber extends ErrorCachingQuerySubscriber { + @Override + public void onResult(int numberOfFields) { + } + + @Override + public void onRecord() { + } + + @Override + public void onField(int offset, AnyValue value) { + } + + @Override + public void onRecordCompleted() { + } + + @Override + public void onResultCompleted(QueryStatistics statistics) { + } + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java index 20a3dc05aec..27760cc1de5 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeMergeSort.java @@ -20,7 +20,7 @@ package org.neo4j.gds.core.utils.paged; import org.jetbrains.annotations.Nullable; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import java.util.concurrent.CountedCompleter; @@ -30,7 +30,7 @@ public final class HugeMergeSort { public static void sort(HugeLongArray array, int concurrency) { var temp = HugeLongArray.newArray(array.size()); - var forkJoinPool = Pools.createForkJoinPool(concurrency); + var forkJoinPool = ExecutorServiceUtil.createForkJoinPool(concurrency); try { forkJoinPool.invoke(new MergeSortTask(null, array, temp, 0, array.size() - 1)); } finally { diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java index 62c0f10ec22..dc4b1db032f 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java @@ -48,7 +48,11 @@ public static Builder builder(int concurrency) { } public static BatchedBuilder batchedBuilder(int concurrency) { - return new BatchedBuilder(concurrency); + return batchedBuilder(concurrency, false); + } + + public static BatchedBuilder batchedBuilder(int concurrency, boolean overrideIds) { + return new BatchedBuilder(concurrency, overrideIds); } private ShardedLongLongMap( @@ -273,7 +277,7 @@ public static final class BatchedBuilder { private final int shardShift; private final int shardMask; - BatchedBuilder(int concurrency) { + BatchedBuilder(int concurrency, boolean overrideIds) { this.nodeCount = new AtomicLong(); int numberOfShards = numberOfShards(concurrency); this.shardShift = Long.SIZE - Integer.numberOfTrailingZeros(numberOfShards); @@ -281,11 +285,16 @@ public static final class BatchedBuilder { this.shards = IntStream.range(0, numberOfShards) .mapToObj(__ -> new Shard()) .toArray(Shard[]::new); - this.batches = CloseableThreadLocal.withInitial(() -> new Batch( - this.shards, - this.shardShift, - this.shardMask - )); + this.batches = CloseableThreadLocal.withInitial(() -> { + if (overrideIds) { + return new OverridingBatch(this.shards, this.shardShift, this.shardMask); + } + return new Batch( + this.shards, + this.shardShift, + this.shardMask + ); + }); } public Batch prepareBatch(int nodeCount) { @@ -316,7 +325,7 @@ public ShardedLongLongMap build(long maxOriginalId) { ); } - public static final class Batch implements IdMapAllocator { + public static class Batch implements IdMapAllocator { private final Shard[] shards; private final int shardShift; @@ -359,6 +368,24 @@ void initBatch(long startId, int length) { } } + /** + * An allocator/batch that overrides the incoming node ids with the generated mapped ids. + */ + private static final class OverridingBatch extends Batch { + + private OverridingBatch(Shard[] shards, int shardShift, int shardMask) { + super(shards, shardShift, shardMask); + } + + @Override + public void insert(long[] nodeIds) { + int length = allocatedSize(); + for (int i = 0; i < length; i++) { + nodeIds[i] = addNode(nodeIds[i]); + } + } + } + private static final class Shard extends MapShard { void addNode(long nodeId, long mappedId) { diff --git a/core/src/test/java/org/neo4j/gds/TerminationTest.java b/core/src/test/java/org/neo4j/gds/TerminationTest.java index 0a176c4714f..26ac0b67165 100644 --- a/core/src/test/java/org/neo4j/gds/TerminationTest.java +++ b/core/src/test/java/org/neo4j/gds/TerminationTest.java @@ -22,8 +22,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.graphdb.TransactionFailureException; import org.neo4j.kernel.api.exceptions.Status; import org.neo4j.kernel.api.query.ExecutingQuery; @@ -103,7 +103,7 @@ private void executeAndKill() { }); // submit - ParallelUtil.run(runnables, Pools.DEFAULT); + ParallelUtil.run(runnables, DefaultPool.INSTANCE); } @Test diff --git a/core/src/test/java/org/neo4j/gds/config/MutatePropertyConfigTest.java b/core/src/test/java/org/neo4j/gds/config/MutateNodePropertyConfigTest.java similarity index 78% rename from core/src/test/java/org/neo4j/gds/config/MutatePropertyConfigTest.java rename to core/src/test/java/org/neo4j/gds/config/MutateNodePropertyConfigTest.java index eef6481ab75..fa43a25bfe4 100644 --- a/core/src/test/java/org/neo4j/gds/config/MutatePropertyConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/MutateNodePropertyConfigTest.java @@ -27,19 +27,23 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; -class MutatePropertyConfigTest { +class MutateNodePropertyConfigTest { @Test void testMutateFailsOnExistingToken() { var graphStore = GdlFactory.of("(a {foo: 42})").build(); - MutatePropertyConfig config = TestMutatePropertyConfigImpl.builder().mutateProperty("foo").build(); + MutateNodePropertyConfig config = TestMutatePropertyConfigImpl.builder().mutateProperty("foo").build(); - assertThatThrownBy(() -> config.validateMutateProperty(graphStore, config.nodeLabelIdentifiers(graphStore), List.of())) + assertThatThrownBy(() -> config.validateMutateProperty( + graphStore, + config.nodeLabelIdentifiers(graphStore), + List.of() + )) .hasMessageContaining("Node property `foo` already exists in the in-memory graph."); } @Configuration - interface TestMutatePropertyConfig extends AlgoBaseConfig, MutatePropertyConfig { + interface TestMutatePropertyConfig extends AlgoBaseConfig, MutateNodePropertyConfig { } } diff --git a/core/src/test/java/org/neo4j/gds/config/MutateRelationshipPropertyConfigTest.java b/core/src/test/java/org/neo4j/gds/config/MutateRelationshipPropertyConfigTest.java new file mode 100644 index 00000000000..9afe4b2fa8f --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/config/MutateRelationshipPropertyConfigTest.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.config; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.annotation.Configuration; +import org.neo4j.gds.gdl.GdlFactory; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class MutateRelationshipPropertyConfigTest { + + @Test + void testMutateFailsOnExistingToken() { + var graphStore = GdlFactory.of("(a {foo: 42})").build(); + TestMutateRelationshipPropertyConfig config = TestMutateRelationshipPropertyConfigImpl.builder().mutateProperty("foo").build(); + + assertThatNoException().isThrownBy(() -> config.graphStoreValidation( + graphStore, + config.nodeLabelIdentifiers(graphStore), + List.of() + )); + + } + + @Configuration + interface TestMutateRelationshipPropertyConfig extends AlgoBaseConfig, MutateRelationshipPropertyConfig { + + } +} diff --git a/core/src/test/java/org/neo4j/gds/core/LoadingTest.java b/core/src/test/java/org/neo4j/gds/core/LoadingTest.java index ab9df441ebe..e81e505edb6 100644 --- a/core/src/test/java/org/neo4j/gds/core/LoadingTest.java +++ b/core/src/test/java/org/neo4j/gds/core/LoadingTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.BaseTest; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import java.util.Arrays; @@ -59,7 +59,7 @@ void setupGraphDb() { void testBasicLoading() { Graph graph = new StoreLoaderBuilder() .databaseService(db) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .addNodeLabel("Node") .addRelationshipType("TYPE") .build() diff --git a/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java b/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java new file mode 100644 index 00000000000..bc41af57a78 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/OpenGdsIdMapBehaviorTest.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.core.loading.ArrayIdMapBuilder; +import org.neo4j.gds.core.loading.GrowingArrayIdMapBuilder; + +import java.util.Optional; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class OpenGdsIdMapBehaviorTest { + + static Stream idMapBuildersById() { + return Stream.of( + Arguments.of(ArrayIdMapBuilder.ID, Optional.empty(), Optional.empty(), GrowingArrayIdMapBuilder.class), + Arguments.of(ArrayIdMapBuilder.ID, Optional.empty(), Optional.of(42L), ArrayIdMapBuilder.class), + Arguments.of(IdMap.NO_TYPE, Optional.empty(), Optional.empty(), GrowingArrayIdMapBuilder.class), + Arguments.of(IdMap.NO_TYPE, Optional.empty(), Optional.of(42L), ArrayIdMapBuilder.class) + ); + } + + @ParameterizedTest + @MethodSource("idMapBuildersById") + void shouldCreateIdMapBuilderById( + String id, + Optional maxOriginalId, + Optional nodeCount, + Class idMapBuilderClazz + ) { + var idMapBuilder = new OpenGdsIdMapBehavior().create(id, 1, maxOriginalId, nodeCount); + assertThat(idMapBuilder).isInstanceOf(idMapBuilderClazz); + } + +} diff --git a/core/src/test/java/org/neo4j/gds/core/compression/common/AdjacencyCompressionTest.java b/core/src/test/java/org/neo4j/gds/core/compression/common/AdjacencyCompressionTest.java index 11d26bac8cd..303b2422a1f 100644 --- a/core/src/test/java/org/neo4j/gds/core/compression/common/AdjacencyCompressionTest.java +++ b/core/src/test/java/org/neo4j/gds/core/compression/common/AdjacencyCompressionTest.java @@ -19,10 +19,8 @@ */ package org.neo4j.gds.core.compression.common; -import org.apache.commons.lang3.mutable.MutableInt; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.compress.LongArrayBuffer; import org.neo4j.gds.core.Aggregation; @@ -31,7 +29,6 @@ import java.util.stream.Stream; import static java.lang.Double.doubleToLongBits; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; @@ -55,7 +52,9 @@ void shouldCountRelationships( AdjacencyCompression.applyDeltaEncoding( data, - new long[][][]{new long[][]{unsortedProperties[0]}, new long[][]{unsortedProperties[1]}}, + new long[][]{ + unsortedProperties[0], unsortedProperties[1] + }, sortedProperties, aggregations, false @@ -78,25 +77,6 @@ void shouldCountRelationships( assertEquals(4L, data.buffer[1]); } - @ParameterizedTest - @CsvSource( - { - "0, 0, 0", - "9, 0, 9", - "10, 1, 0", - "41, 1, 31", - "42, 2, 0", - } - ) - void findPosition(int position, int expectedPageIndex, int expectedIndexInPage) { - int[] lengths = {10, 42, 1337}; - var pageIndex = new MutableInt(); - var indexInPage = new MutableInt(); - AdjacencyCompression.findPosition(lengths, position, pageIndex, indexInPage); - assertThat(pageIndex.getValue()).isEqualTo(expectedPageIndex); - assertThat(indexInPage.getValue()).isEqualTo(expectedIndexInPage); - } - static Stream aggregationsWithResults() { return Stream.of( Arguments.of( @@ -106,7 +86,13 @@ static Stream aggregationsWithResults() { Aggregation.COUNT, Aggregation.COUNT }, - new double[][]{{4d, 2d}, {2d, 1d}}, + new double[][]{ + { + 4d, 2d + }, { + 2d, 1d + } + }, "COUNT" ), Arguments.of( @@ -116,7 +102,13 @@ static Stream aggregationsWithResults() { Aggregation.SUM, Aggregation.SUM }, - new double[][]{{16d, 8d}, {27d, 14d}}, + new double[][]{ + { + 16d, 8d + }, { + 27d, 14d + } + }, "SUM" ), Arguments.of( @@ -126,7 +118,13 @@ static Stream aggregationsWithResults() { Aggregation.MIN, Aggregation.MIN }, - new double[][]{{2d, 3d}, {4d, 6d}}, + new double[][]{ + { + 2d, 3d + }, { + 4d, 6d + } + }, "MIN" ), Arguments.of( @@ -136,50 +134,62 @@ static Stream aggregationsWithResults() { Aggregation.MAX, Aggregation.MAX }, - new double[][]{{5d, 5d}, {8d, 8d}}, + new double[][]{ + { + 5d, 5d + }, { + 8d, 8d + } + }, "MAX" ) ); } private static long[][] weights() { - return new long[][]{{ - doubleToLongBits(2), - doubleToLongBits(4), - doubleToLongBits(3), - doubleToLongBits(5), - doubleToLongBits(5), - doubleToLongBits(5) - }, { - doubleToLongBits(4), - doubleToLongBits(7), - doubleToLongBits(6), - doubleToLongBits(8), - doubleToLongBits(8), - doubleToLongBits(8) - }}; + return new long[][]{ + { + doubleToLongBits(2), + doubleToLongBits(4), + doubleToLongBits(3), + doubleToLongBits(5), + doubleToLongBits(5), + doubleToLongBits(5) + }, { + doubleToLongBits(4), + doubleToLongBits(7), + doubleToLongBits(6), + doubleToLongBits(8), + doubleToLongBits(8), + doubleToLongBits(8) + } + }; } private static long[][] countWeights() { - return new long[][]{{ - doubleToLongBits(1.0), - doubleToLongBits(1.0), - doubleToLongBits(1.0), - doubleToLongBits(1.0), - doubleToLongBits(1.0), - doubleToLongBits(1.0) - }, { - doubleToLongBits(1.0), - doubleToLongBits(0.0), - doubleToLongBits(1.0), - doubleToLongBits(0.0), - doubleToLongBits(1.0), - doubleToLongBits(0.0) - }}; + return new long[][]{ + { + doubleToLongBits(1.0), + doubleToLongBits(1.0), + doubleToLongBits(1.0), + doubleToLongBits(1.0), + doubleToLongBits(1.0), + doubleToLongBits(1.0) + }, { + doubleToLongBits(1.0), + doubleToLongBits(0.0), + doubleToLongBits(1.0), + doubleToLongBits(0.0), + doubleToLongBits(1.0), + doubleToLongBits(0.0) + } + }; } private static long[] values() { - return new long[]{1, 1, 5, 5, 1, 1}; + return new long[]{ + 1, 1, 5, 5, 1, 1 + }; } } diff --git a/core/src/test/java/org/neo4j/gds/core/compression/packed/AdjacencyPackerTest.java b/core/src/test/java/org/neo4j/gds/core/compression/packed/AdjacencyPackerTest.java index 93ce44d10f2..5017b92d042 100644 --- a/core/src/test/java/org/neo4j/gds/core/compression/packed/AdjacencyPackerTest.java +++ b/core/src/test/java/org/neo4j/gds/core/compression/packed/AdjacencyPackerTest.java @@ -160,7 +160,7 @@ void loopPack5Test() { @Test void loopPack64Test() { int bits = 64; - long upperBound = (1L << bits) - 1; + long upperBound = -1L; long[] data = LongStream.concat(LongStream.rangeClosed(0, 42), LongStream.of(upperBound)).toArray(); int length = data.length; int requiredBytes = BitUtil.ceilDiv(length * bits, Byte.SIZE); diff --git a/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java b/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java index 70d0248b2da..d7fb8ab1200 100644 --- a/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java +++ b/core/src/test/java/org/neo4j/gds/core/compression/varlong/TransientCsrListTest.java @@ -31,7 +31,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.TestMethodRunner; import org.neo4j.gds.core.compression.packed.PackedAdjacencyList; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.construction.GraphFactory; import java.util.Arrays; @@ -438,7 +438,7 @@ static AdjacencyList adjacencyListFromTargets(IdMap idMap, long[] targets) { .nodes(idMap) .relationshipType(RelationshipType.of("REL")) .concurrency(1) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); Arrays.stream(targets).forEach(target -> relationshipsBuilder.add(sourceNodeId, target)); diff --git a/core/src/test/java/org/neo4j/gds/core/concurrency/ParallelUtilTest.java b/core/src/test/java/org/neo4j/gds/core/concurrency/ParallelUtilTest.java index cc73aa2be24..a385bafdf33 100644 --- a/core/src/test/java/org/neo4j/gds/core/concurrency/ParallelUtilTest.java +++ b/core/src/test/java/org/neo4j/gds/core/concurrency/ParallelUtilTest.java @@ -358,7 +358,7 @@ void shouldBailOnTermination() { AtomicReference thrownException = new AtomicReference<>(); AtomicBoolean running = new AtomicBoolean(true); TerminationFlag isRunning = running::get; - var thread = Pools.newThread(() -> tasks.run(t -> + var thread = ExecutorServiceUtil.newThread(() -> tasks.run(t -> RunWithConcurrency.builder() .concurrency(2) .tasks(t) diff --git a/core/src/test/java/org/neo4j/gds/core/concurrency/PoolsTest.java b/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java similarity index 69% rename from core/src/test/java/org/neo4j/gds/core/concurrency/PoolsTest.java rename to core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java index 3e88180c7ac..635f2ed63b4 100644 --- a/core/src/test/java/org/neo4j/gds/core/concurrency/PoolsTest.java +++ b/core/src/test/java/org/neo4j/gds/core/concurrency/PoolSizesServiceTest.java @@ -20,19 +20,18 @@ package org.neo4j.gds.core.concurrency; import org.junit.jupiter.api.Test; +import org.neo4j.gds.concurrency.PoolSizesService; -import java.util.concurrent.ThreadPoolExecutor; +import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; - -class PoolsTest { +class PoolSizesServiceTest { @Test - void shouldGetLimitedPool() { - ThreadPoolExecutor defaultPool = (ThreadPoolExecutor) Pools.createDefaultPool(); + void openGdsPoolSizesByDefault() { + var poolSizes = PoolSizesService.poolSizes(); - assertEquals(4, defaultPool.getCorePoolSize()); - assertEquals(4, defaultPool.getMaximumPoolSize()); + assertThat(poolSizes.corePoolSize()).isEqualTo(4); + assertThat(poolSizes.maxPoolSize()).isEqualTo(4); } } diff --git a/core/src/test/java/org/neo4j/gds/core/huge/CompositeAdjacencyListTest.java b/core/src/test/java/org/neo4j/gds/core/huge/CompositeAdjacencyListTest.java index 2b2751cbcf5..cd3abfed391 100644 --- a/core/src/test/java/org/neo4j/gds/core/huge/CompositeAdjacencyListTest.java +++ b/core/src/test/java/org/neo4j/gds/core/huge/CompositeAdjacencyListTest.java @@ -20,11 +20,12 @@ package org.neo4j.gds.core.huge; import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.Graph; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.Inject; -import org.neo4j.gds.api.Graph; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @GdlExtension @@ -47,4 +48,26 @@ void shouldIgnoreInputDegreeForCursor() { var cursor = adjacencyList.adjacencyCursor(0); assertEquals(2, cursor.remaining()); } + + @Test + void shouldReuseCursor() { + + var adjacencyList = ((UnionGraph) graph).relationshipTopology(); + var cursor = adjacencyList.adjacencyCursor(null, 0); + assertThat(cursor.peekVLong()).isEqualTo(1L); + cursor.nextVLong(); + + cursor = adjacencyList.adjacencyCursor(cursor, 0); + assertThat(cursor.peekVLong()).isEqualTo(1L); + + + cursor.nextVLong(); + cursor.nextVLong(); + + cursor = adjacencyList.adjacencyCursor(cursor, 0); + assertThat(cursor.hasNextVLong()).isTrue(); + + + } + } diff --git a/core/src/test/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursorTest.java b/core/src/test/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursorTest.java index e6aa2010a06..d0cf4b8a62c 100644 --- a/core/src/test/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursorTest.java +++ b/core/src/test/java/org/neo4j/gds/core/huge/NodeFilteredAdjacencyCursorTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.AdjacencyCursor; +import org.neo4j.gds.api.FilteredIdMap; import java.util.ArrayList; import java.util.List; @@ -78,7 +79,7 @@ void shouldAdvanceWithFilter() { assertThat(adjacencyCursor.advance(9L)).isEqualTo(AdjacencyCursor.NOT_FOUND); } - static class FilteredDirectIdMap extends DirectIdMap { + static class FilteredDirectIdMap extends DirectIdMap implements FilteredIdMap { private final LongPredicate nodeFilter; @@ -91,5 +92,15 @@ static class FilteredDirectIdMap extends DirectIdMap { public boolean containsOriginalId(long originalNodeId) { return nodeFilter.test(originalNodeId); } + + @Override + public long toFilteredNodeId(long rootNodeId) { + return rootNodeId; + } + + @Override + public boolean containsRootNodeId(long rootNodeId) { + return nodeFilter.test(rootNodeId); + } } } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java b/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java index e2af86d88f5..c8e353a7a2c 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/ChunkedAdjacencyListsTest.java @@ -23,13 +23,10 @@ import org.neo4j.gds.core.compression.common.AdjacencyCompression; import org.neo4j.gds.core.compression.common.ZigZagLongDecoding; -import java.util.Arrays; - import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.neo4j.gds.core.compression.common.ZigZagLongDecoding.Identity.INSTANCE; import static org.neo4j.gds.core.loading.AdjacencyPreAggregation.IGNORE_VALUE; -import static org.neo4j.gds.core.loading.ChunkedAdjacencyLists.NEXT_CHUNK_LENGTH; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; class ChunkedAdjacencyListsTest { @@ -38,19 +35,25 @@ class ChunkedAdjacencyListsTest { void shouldWriteSingleTargetList() { var adjacencyLists = ChunkedAdjacencyLists.of(0, 0); - var input = new long[]{42L, 1337L, 5L}; + var input = new long[]{ + 42L, 1337L, 5L + }; adjacencyLists.add(0, input, 0, 3, 3); - var expectedTargets = new long[]{42L, 1337L, 5L}; + var expectedTargets = new long[]{ + 42L, 1337L, 5L + }; var actualTargets = new long[3]; - adjacencyLists.consume((nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( - actualTargets, - targets, - length, - position, - INSTANCE - )); + adjacencyLists.consume( + (nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( + actualTargets, + targets, + length, + position, + INSTANCE + ) + ); assertThat(actualTargets).containsExactly(expectedTargets); } @@ -58,109 +61,50 @@ void shouldWriteSingleTargetList() { void shouldWriteMultipleTimesIntoTargetList() { var adjacencyLists = ChunkedAdjacencyLists.of(0, 0); - adjacencyLists.add(0, new long[]{42L, 1337L, 5L}, 0, 3, 3); - adjacencyLists.add(0, new long[]{42L, 1337L, 5L}, 1, 3, 2); + adjacencyLists.add(0, new long[]{ + 42L, 1337L, 5L + }, 0, 3, 3); + adjacencyLists.add(0, new long[]{ + 42L, 1337L, 5L + }, 1, 3, 2); - var expectedTargets = new long[]{42L, 1337L, 5L, 1337L, 5L}; + var expectedTargets = new long[]{ + 42L, 1337L, 5L, 1337L, 5L + }; var actualTargets = new long[5]; - adjacencyLists.consume((nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( - actualTargets, - targets, - length, - position, - INSTANCE - )); - assertThat(actualTargets).containsExactly(expectedTargets); - } - - @Test - void shouldWriteLargeAdjacencyListsWithOverflow() { - var adjacencyLists = ChunkedAdjacencyLists.of(1, 0); - - var smallAdjacency = new long[37]; - Arrays.setAll(smallAdjacency, i -> i); - - var largeAdjacency = new long[NEXT_CHUNK_LENGTH[NEXT_CHUNK_LENGTH.length - 1] + 100]; - Arrays.setAll(largeAdjacency, i -> i + 42L); - - var smallProperties = new long[37]; - Arrays.setAll(smallProperties, i -> i); - - var largeProperties = new long[NEXT_CHUNK_LENGTH[NEXT_CHUNK_LENGTH.length - 1] + 100]; - Arrays.setAll(largeProperties, i -> i + 42L); - - adjacencyLists.add( - 0, - Arrays.copyOf(smallAdjacency, smallAdjacency.length), - new long[][]{smallProperties}, - 0, - smallAdjacency.length, - smallAdjacency.length - ); - adjacencyLists.add( - 0, - Arrays.copyOf(smallAdjacency, smallAdjacency.length), - new long[][]{smallProperties}, - 0, - smallAdjacency.length, - smallAdjacency.length - ); - adjacencyLists.add( - 0, - Arrays.copyOf(largeAdjacency, largeAdjacency.length), - new long[][]{largeProperties}, - 0, - largeAdjacency.length, - largeAdjacency.length - ); - - var expectedTargets = new long[smallAdjacency.length * 2 + largeAdjacency.length]; - System.arraycopy(smallAdjacency, 0, expectedTargets, 0, smallAdjacency.length); - System.arraycopy(smallAdjacency, 0, expectedTargets, smallAdjacency.length, smallAdjacency.length); - System.arraycopy(largeAdjacency, 0, expectedTargets, smallAdjacency.length * 2, largeAdjacency.length); - - var expectedProperties = new long[smallProperties.length * 2 + largeProperties.length]; - System.arraycopy(smallProperties, 0, expectedProperties, 0, smallProperties.length); - System.arraycopy(smallProperties, 0, expectedProperties, smallAdjacency.length, smallProperties.length); - System.arraycopy(largeProperties, 0, expectedProperties, smallAdjacency.length * 2, largeProperties.length); - - var actualTargets = new long[smallAdjacency.length * 2 + largeAdjacency.length]; - adjacencyLists.consume((nodeId, targets, properties, position, length) -> { - AdjacencyCompression.zigZagUncompressFrom( + adjacencyLists.consume( + (nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( actualTargets, targets, length, position, INSTANCE - ); - - long[] actualProperties = new long[expectedProperties.length]; - var written = 0; - for (long[] propertyChunk : properties[0]) { - var valuesToCopy = Math.min(propertyChunk.length, length - written); - System.arraycopy(propertyChunk, 0, actualProperties, written, valuesToCopy); - written += valuesToCopy; - } - assertThat(Arrays.compare(expectedProperties, actualProperties)).isEqualTo(0); - }); - assertThat(Arrays.compare(expectedTargets, actualTargets)).isEqualTo(0); + ) + ); + assertThat(actualTargets).containsExactly(expectedTargets); } @Test void shouldWriteWithProperties() { var adjacencyLists = ChunkedAdjacencyLists.of(2, 0); - var input = new long[]{42L, 1337L, 5L, 6L}; - var properties = new long[][]{{42L, 1337L, 5L, 6L}, {8L, 8L, 8L, 8L}}; + var input = new long[]{ + 42L, 1337L, 5L, 6L + }; + var properties = new long[][]{ + { + 42L, 1337L, 5L, 6L + }, { + 8L, 8L, 8L, 8L + } + }; adjacencyLists.add(0, input, properties, 0, 4, 4); adjacencyLists.consume((nodeId, targets, actualProperties, position, length) -> { assertThat(actualProperties).hasNumberOfRows(2); - assertThat(actualProperties[0]).hasNumberOfRows(1); - assertThat(actualProperties[1]).hasNumberOfRows(1); - assertThat(actualProperties[0][0]).containsSequence(42L, 1337L, 5L, 6L); - assertThat(actualProperties[1][0]).containsSequence(8L, 8L, 8L, 8L); + assertThat(actualProperties[0]).containsSequence(42L, 1337L, 5L, 6L); + assertThat(actualProperties[1]).containsSequence(8L, 8L, 8L, 8L); }); } @@ -168,17 +112,23 @@ void shouldWriteWithProperties() { void shouldWriteWithPropertiesWithOffset() { var adjacencyLists = ChunkedAdjacencyLists.of(2, 0); - var input = new long[]{13L, 37L, 42L, 1337L, 5L, 6L}; - var properties = new long[][]{{0L, 0L, 42L, 1337L, 5L, 6L}, {0L, 0L, 8L, 8L, 8L, 8L}}; + var input = new long[]{ + 13L, 37L, 42L, 1337L, 5L, 6L + }; + var properties = new long[][]{ + { + 0L, 0L, 42L, 1337L, 5L, 6L + }, { + 0L, 0L, 8L, 8L, 8L, 8L + } + }; adjacencyLists.add(0, input, properties, 2, 6, 4); adjacencyLists.consume((nodeId, targets, actualProperties, position, length) -> { assertThat(actualProperties).hasNumberOfRows(2); - assertThat(actualProperties[0]).hasNumberOfRows(1); - assertThat(actualProperties[1]).hasNumberOfRows(1); - assertThat(actualProperties[0][0]).containsSequence(42L, 1337L, 5L, 6L); - assertThat(actualProperties[1][0]).containsSequence(8L, 8L, 8L, 8L); + assertThat(actualProperties[0]).containsSequence(42L, 1337L, 5L, 6L); + assertThat(actualProperties[1]).containsSequence(8L, 8L, 8L, 8L); }); } @@ -186,12 +136,18 @@ void shouldWriteWithPropertiesWithOffset() { void shouldAllowConsumptionOfAllElements() { var adjacencyLists = ChunkedAdjacencyLists.of(0, 0); - adjacencyLists.add(1, new long[]{42L, 1337L, 5L}, 0, 3, 3); - adjacencyLists.add(8, new long[]{1L, 2L}, 0, 2, 2); + adjacencyLists.add(1, new long[]{ + 42L, 1337L, 5L + }, 0, 3, 3); + adjacencyLists.add(8, new long[]{ + 1L, 2L + }, 0, 2, 2); // Skip 2 pages var largeIndex = 3 * 4096 + 1; - adjacencyLists.add(largeIndex, new long[]{42L, 42L}, 0, 2, 2); + adjacencyLists.add(largeIndex, new long[]{ + 42L, 42L + }, 0, 2, 2); adjacencyLists.consume((id, targets, properties, compressedBytesSize, compressedTargets) -> { assertThat(properties).isNull(); @@ -221,19 +177,25 @@ void shouldAllowConsumptionOfAllElements() { void addWithPreAggregation() { var adjacencyLists = ChunkedAdjacencyLists.of(0, 0); - var input = new long[]{42L, IGNORE_VALUE, IGNORE_VALUE, 1337L, 5L}; + var input = new long[]{ + 42L, IGNORE_VALUE, IGNORE_VALUE, 1337L, 5L + }; adjacencyLists.add(0, input, 0, 5, 3); - var expectedTargets = new long[]{42L, 1337L, 5L}; + var expectedTargets = new long[]{ + 42L, 1337L, 5L + }; var actualTargets = new long[3]; - adjacencyLists.consume((nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( - actualTargets, - targets, - length, - position, - INSTANCE - )); + adjacencyLists.consume( + (nodeId, targets, __, position, length) -> AdjacencyCompression.zigZagUncompressFrom( + actualTargets, + targets, + length, + position, + INSTANCE + ) + ); assertThat(actualTargets).containsExactly(expectedTargets); } @@ -241,11 +203,19 @@ void addWithPreAggregation() { void addWithPreAggregatedWeights() { var adjacencyLists = ChunkedAdjacencyLists.of(1, 0); - var input = new long[]{42L, IGNORE_VALUE, 1337L, 5L}; - var properties = new long[][]{{3L, 2L, 3L, 4L}}; + var input = new long[]{ + 42L, IGNORE_VALUE, 1337L, 5L + }; + var properties = new long[][]{ + { + 3L, 2L, 3L, 4L + } + }; adjacencyLists.add(0, input, properties, 0, 4, 3); - var expectedTargets = new long[]{42L, 1337L, 5L}; + var expectedTargets = new long[]{ + 42L, 1337L, 5L + }; adjacencyLists.consume((nodeId, targets, actualProperties, position, length) -> { var actualTargets = new long[3]; @@ -259,8 +229,7 @@ void addWithPreAggregatedWeights() { assertThat(actualTargets).containsExactly(expectedTargets); assertThat(actualProperties).hasNumberOfRows(1); - assertThat(actualProperties[0]).hasNumberOfRows(1); - assertThat(actualProperties[0][0]).containsSequence(3L, 3L, 4L); + assertThat(actualProperties[0]).containsSequence(3L, 3L, 4L); }); } } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java index ccaa102680a..ca3179597b1 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java @@ -78,29 +78,34 @@ void setUp() { void testLoadCypher() { clearDb(); String query = " CREATE (n1 {partition: 6})-[:REL {prop: 1}]->(n2 {foo: 4.0})-[:REL {prop: 2}]->(n3)" + - " CREATE (n1)-[:REL {prop: 3}]->(n3)" + - " RETURN id(n1) AS id1, id(n2) AS id2, id(n3) AS id3"; + " CREATE (n1)-[:REL {prop: 3}]->(n3)" + + " RETURN id(n1) AS id1, id(n2) AS id2, id(n3) AS id3"; runQuery(query); String nodes = "MATCH (n) RETURN id(n) AS id, COALESCE(n.partition, 0) AS partition, COALESCE(n.foo, 5.0) AS foo"; String rels = "MATCH (n)-[r]->(m) WHERE type(r) = 'REL' " + - "RETURN id(n) AS source, id(m) AS target, coalesce(head(collect(r.prop)), 0)"; + "RETURN id(n) AS source, id(m) AS target, coalesce(head(collect(r.prop)), 0)"; - Graph graph = applyInTransaction(db, tx -> new CypherLoaderBuilder().databaseService(db) + Graph graph = applyInTransaction( + db, + tx -> new CypherLoaderBuilder().databaseService(db) .nodeQuery(nodes) .relationshipQuery(rels) .build() .graph() ); - assertGraphEquals(fromGdl( - "(a {partition: 6, foo: 5.0})" + - "(b {partition: 0, foo: 4.0})" + - "(c {partition: 0, foo: 5.0})" + - "(a)-[{w:1.0}]->(b)" + - "(a)-[{w:3.0}]->(c)" + - "(b)-[{w:2.0}]->(c)" - ), graph); + assertGraphEquals( + fromGdl( + "(a {partition: 6, foo: 5.0})" + + "(b {partition: 0, foo: 4.0})" + + "(c {partition: 0, foo: 5.0})" + + "(a)-[{w:1.0}]->(b)" + + "(a)-[{w:3.0}]->(c)" + + "(b)-[{w:2.0}]->(c)" + ), + graph + ); } @Test @@ -164,8 +169,8 @@ void doubleListNodeProperty() { @Test void doubleListWithEmptyList() { var nodeQuery = "WITH [0, 1] AS ids, [[1.3, 3.7], []] AS properties " + - "UNWIND ids AS id " + - "RETURN id, properties[id] AS list"; + "UNWIND ids AS id " + + "RETURN id, properties[id] AS list"; var builder = new CypherLoaderBuilder() .databaseService(db) @@ -180,8 +185,8 @@ void doubleListWithEmptyList() { @Test void longListWithEmptyList() { var nodeQuery = "WITH [0, 1] AS ids, [[1, 3, 3, 7], []] AS properties " + - "UNWIND ids AS id " + - "RETURN id, properties[id] AS list"; + "UNWIND ids AS id " + + "RETURN id, properties[id] AS list"; var builder = new CypherLoaderBuilder() .databaseService(db) @@ -269,9 +274,9 @@ void testMultipleNodeProperties() { clearDb(); runQuery( "CREATE" + - " ({prop1: 1})" + - ", ({prop2: 2})" + - ", ({prop3: 3})" + " ({prop1: 1})" + + ", ({prop2: 2})" + + ", ({prop3: 3})" ); PropertyMapping prop1 = PropertyMapping.of("prop1", 0); PropertyMapping prop2 = PropertyMapping.of("prop2", 0); @@ -282,8 +287,8 @@ void testMultipleNodeProperties() { .graph(); String gdl = "(a {prop1: 1, prop2: 0, prop3: 0})" + - "(b {prop1: 0, prop2: 2, prop3: 0})" + - "(c {prop1: 0, prop2: 0, prop3: 3})"; + "(b {prop1: 0, prop2: 2, prop3: 0})" + + "(c {prop1: 0, prop2: 0, prop3: 3})"; assertGraphEquals(fromGdl(gdl), graph); } @@ -293,11 +298,11 @@ void testMultipleRelationshipProperties() { clearDb(); runQuery( "CREATE" + - " (n1)" + - ", (n2)" + - ", (n1)-[:REL {prop1: 1.0}]->(n2)" + - ", (n1)-[:REL {prop2: 2.0}]->(n2)" + - ", (n1)-[:REL {prop3: 3.0}]->(n2)" + " (n1)" + + ", (n2)" + + ", (n1)-[:REL {prop1: 1.0}]->(n2)" + + ", (n1)-[:REL {prop2: 2.0}]->(n2)" + + ", (n1)-[:REL {prop3: 3.0}]->(n2)" ); PropertyMapping prop1 = PropertyMapping.of("prop1", 0D); PropertyMapping prop2 = PropertyMapping.of("prop2", 0D); @@ -308,23 +313,43 @@ void testMultipleRelationshipProperties() { .withDefaultAggregation(Aggregation.DEFAULT) .graphStore(); - String expectedGraph = - " (a)-[{w: %f}]->(b)" + + String expectedGraph = " (a)-[{w: %f}]->(b)" + ", (a)-[{w: %f}]->(b)" + ", (a)-[{w: %f}]->(b)"; assertGraphEquals( - fromGdl(formatWithLocale(expectedGraph, 1.0f, prop1.defaultValue().doubleValue(), prop1.defaultValue().doubleValue())), + fromGdl( + formatWithLocale( + expectedGraph, + 1.0f, + prop1.defaultValue().doubleValue(), + prop1.defaultValue().doubleValue() + ) + ), graphs.getGraph(ALL_RELATIONSHIPS, Optional.of(prop1.propertyKey())) ); assertGraphEquals( - fromGdl(formatWithLocale(expectedGraph, prop2.defaultValue().doubleValue(), 2.0, prop2.defaultValue().doubleValue())), + fromGdl( + formatWithLocale( + expectedGraph, + prop2.defaultValue().doubleValue(), + 2.0, + prop2.defaultValue().doubleValue() + ) + ), graphs.getGraph(ALL_RELATIONSHIPS, Optional.of(prop2.propertyKey())) ); assertGraphEquals( - fromGdl(formatWithLocale(expectedGraph, prop3.defaultValue().doubleValue(), prop3.defaultValue().doubleValue(), 3.0)), + fromGdl( + formatWithLocale( + expectedGraph, + prop3.defaultValue().doubleValue(), + prop3.defaultValue().doubleValue(), + 3.0 + ) + ), graphs.getGraph(ALL_RELATIONSHIPS, Optional.of(prop3.propertyKey())) ); } @@ -347,15 +372,15 @@ void loadGraphWithParameterizedCypherQuery() { void testLoadingGraphWithLabelInformation() { clearDb(); String query = "CREATE" + - " (a:A)" + - ", (b:B)" + - ", (c:C)" + - ", (ab:A:B)" + - "CREATE" + - " (a)-[:REL]->(b)" + - ", (a)-[:REL]->(c)" + - ", (a)-[:REL]->(ab)" + - ", (c)-[:REL]->(a)"; + " (a:A)" + + ", (b:B)" + + ", (c:C)" + + ", (ab:A:B)" + + "CREATE" + + " (a)-[:REL]->(b)" + + ", (a)-[:REL]->(c)" + + ", (a)-[:REL]->(ab)" + + ", (c)-[:REL]->(a)"; runQuery(query); @@ -438,13 +463,15 @@ void canLoadArrayNodeProperties() { String nodes = "MATCH (n) RETURN id(n) AS id, n.longArray as longArray, n.doubleArray as doubleArray"; String rels = "MATCH (n)" + - "RETURN id(n) AS source, id(n) AS target"; + "RETURN id(n) AS source, id(n) AS target"; - Graph graph = applyInTransaction(db, tx -> new CypherLoaderBuilder().databaseService(db) - .nodeQuery(nodes) - .relationshipQuery(rels) - .build() - .graph() + Graph graph = applyInTransaction( + db, + tx -> new CypherLoaderBuilder().databaseService(db) + .nodeQuery(nodes) + .relationshipQuery(rels) + .build() + .graph() ); assertGraphEquals(fromGdl("(a {longArray: [42L], doubleArray: [42.0D]}), (a)-->(a)"), graph); @@ -524,24 +551,24 @@ private static Stream memoryEstimationVariants() { "Topology Only", "MATCH (n) RETURN id(n) as id", "MATCH (n)-[r]->(m) RETURN id(n) AS source, id(m) AS target", - 1202360, - 1202360 + 1202280, + 1202280 ), Arguments.of( "Node properties", "MATCH (n) RETURN id(n) as id, n.id as idProp", "MATCH (n)-[r]->(m) RETURN id(n) AS source, id(m) AS target", - 1300800, - 1300800 + 1300720, + 1300720 ), Arguments.of( "Relationship properties", "MATCH (n) RETURN id(n) as id", "MATCH (n)-[r]->(m) RETURN id(n) AS source, id(m) AS target, r.prop as prop", - 1692720, - 1692720 + 1692640, + 1692640 ) ); } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java new file mode 100644 index 00000000000..3b8a8f2feed --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapBuilderTest.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.neo4j.gds.core.idmap.IdMapBuilderTest; + +public class HighLimitIdMapBuilderTest extends IdMapBuilderTest { + + @Override + protected IdMapBuilder builder(long capacity, int concurrency) { + return HighLimitIdMapBuilder.of(concurrency, ArrayIdMapBuilder.of(capacity)); + } +} diff --git a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java index e6301273152..1ca29bc4020 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/HighLimitIdMapTest.java @@ -29,6 +29,8 @@ import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; import java.util.List; +import java.util.Optional; +import java.util.stream.LongStream; import static org.assertj.core.api.Assertions.assertThat; import static org.neo4j.gds.api.IdMap.NOT_FOUND; @@ -68,6 +70,32 @@ void shouldHandleUnmappedIds() { assertThat(highLimitIdMap.containsOriginalId(1337)).isFalse(); } + @Test + void shouldReturnCorrectTypeIds() { + long[] nodes = LongStream.range(0, 42).toArray(); + var builder = HighLimitIdMapBuilder.of(1, ArrayIdMapBuilder.of(nodes.length)); + builder.allocate(nodes.length).insert(nodes); + var idMap = builder.build(LabelInformationBuilders.allNodes(), nodes.length - 1, 1); + + assertThat(idMap.typeId()).contains(HighLimitIdMapBuilder.ID).contains(ArrayIdMapBuilder.ID); + assertThat(HighLimitIdMap.innerTypeId(idMap.typeId()).get()).isEqualTo(ArrayIdMapBuilder.ID); + } + + @Test + void shouldReturnCorrectInnerTypeId() { + assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID)).isEqualTo(Optional.empty()); + assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID + "-foobar")).isEqualTo(Optional.of("foobar")); + assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID + "-" + HighLimitIdMapBuilder.ID)).isEqualTo(Optional.empty()); + assertThat(HighLimitIdMap.innerTypeId("foobar")).isEqualTo(Optional.empty()); + assertThat(HighLimitIdMap.innerTypeId(HighLimitIdMapBuilder.ID + "-")).isEqualTo(Optional.empty()); + } + + @Test + void shouldIdentifyHighLimitIdMaps() { + assertThat(HighLimitIdMap.isHighLimitIdMap(HighLimitIdMapBuilder.ID)).isTrue(); + assertThat(HighLimitIdMap.isHighLimitIdMap("foobar")).isFalse(); + } + @Nested class FilteredHighLimitIdMap { diff --git a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java index 2aa39412604..d5d078c26df 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java @@ -22,8 +22,8 @@ import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -66,7 +66,7 @@ void parallelAddDuplicateNodes() { }, Optional.empty()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var highLimitIdMap = lazyIdMapBuilder.build().idMap(); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NativeFactoryTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NativeFactoryTest.java index a04f324c73a..5f0b456357d 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NativeFactoryTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NativeFactoryTest.java @@ -47,11 +47,13 @@ void memoryEstimationBitMapDisabled() { .build(); var memoryEstimation = new AtomicReference(); - memoryEstimation.set(NativeFactory.getMemoryEstimation( - NodeProjections.all(), - RelationshipProjections.single(RelationshipType.ALL_RELATIONSHIPS, RelationshipProjection.ALL), - false - )); + memoryEstimation.set( + NativeFactory.getMemoryEstimation( + NodeProjections.all(), + RelationshipProjections.single(RelationshipType.ALL_RELATIONSHIPS, RelationshipProjection.ALL), + false + ) + ); var estimate = memoryEstimation.get().estimate(dimensions, 1); @@ -83,8 +85,8 @@ void memoryEstimationForSingleProjection() { .getMemoryEstimation(nodeProjections, relationshipProjections, true) .estimate(dimensions, 1); - assertEquals(6_828_526_824L, estimate.memoryUsage().min); - assertEquals(7_633_833_192L, estimate.memoryUsage().max); + assertEquals(6_828_526_776L, estimate.memoryUsage().min); + assertEquals(7_633_833_144L, estimate.memoryUsage().max); } @Test @@ -112,8 +114,8 @@ void memoryEstimationForIndexedProjection() { .getMemoryEstimation(nodeProjections, relationshipProjections, true) .estimate(dimensions, 1); - assertEquals(12_056_534_496L, estimate.memoryUsage().min); - assertEquals(13_667_147_232L, estimate.memoryUsage().max); + assertEquals(12_056_534_400L, estimate.memoryUsage().min); + assertEquals(13_667_147_136L, estimate.memoryUsage().max); } @Test @@ -135,7 +137,7 @@ void memoryEstimationForMultipleProjections() { .getMemoryEstimation(nodeProjections, relationshipProjections, true) .estimate(dimensions, 1); - assertEquals(12_056_534_496L, estimate.memoryUsage().min); - assertEquals(13_667_147_232L, estimate.memoryUsage().max); + assertEquals(12_056_534_400L, estimate.memoryUsage().min); + assertEquals(13_667_147_136L, estimate.memoryUsage().max); } } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NodesBatchBufferTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NodesBatchBufferTest.java index 84d764b70f7..8360521d23f 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NodesBatchBufferTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NodesBatchBufferTest.java @@ -29,46 +29,6 @@ class NodesBatchBufferTest { - @Test - void shouldIgnoreNodesThatAreOutOfBoundsOnAdd() { - var nodesBatchBuffer = new NodesBatchBufferBuilder() - .capacity(3) - .highestPossibleNodeCount(43) - .build(); - - // within range - nodesBatchBuffer.add(21, Neo4jProxy.noPropertyReference(), new long[0]); - // end of range - nodesBatchBuffer.add(42, Neo4jProxy.noPropertyReference(), new long[0]); - // out of range - nodesBatchBuffer.add(84, Neo4jProxy.noPropertyReference(), new long[0]); - - assertThat(nodesBatchBuffer) - .returns(2, RecordsBatchBuffer::length) - .returns(new long[]{21, 42, 0}, RecordsBatchBuffer::batch); - } - - @Test - void shouldIgnoreNodesThatAreOutOfBoundsOnAddWithLabelInformation() { - var nodesBatchBuffer = new NodesBatchBufferBuilder() - .capacity(3) - .highestPossibleNodeCount(43) - .hasLabelInformation(true) - .nodeLabelIds(LongHashSet.from(0)) - .build(); - - // within range - nodesBatchBuffer.add(21, Neo4jProxy.noPropertyReference(), new long[]{0}); - // end of range - nodesBatchBuffer.add(42, Neo4jProxy.noPropertyReference(), new long[]{0}); - // out of range - nodesBatchBuffer.add(84, Neo4jProxy.noPropertyReference(), new long[]{0}); - - assertThat(nodesBatchBuffer) - .returns(2, RecordsBatchBuffer::length) - .returns(new long[]{21, 42, 0}, RecordsBatchBuffer::batch); - } - @Test void shouldIgnoreNodesThatAreOutOfBoundsOnOffer() { var nodesBatchBuffer = new NodesBatchBufferBuilder() diff --git a/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java b/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java index a310f2078a0..54340a1ddb2 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/ScanningRelationshipsImporterTest.java @@ -37,7 +37,7 @@ import org.neo4j.gds.config.ImmutableGraphProjectFromStoreConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.GraphDimensionsStoreReader; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -152,7 +152,7 @@ private long[] nodeIds(String... nodeVariables) { private GraphLoaderContext graphLoaderContext() { return ImmutableGraphLoaderContext.builder() - .executor(Pools.DEFAULT) + .executor(DefaultPool.INSTANCE) .log(NullLog.getInstance()) .terminationFlag(TerminationFlag.RUNNING_TRUE) .transactionContext(DatabaseTransactionContext.of(db, db.beginTx())) diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java index faa8cc0e289..5e06652895f 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeAtomicGrowingBitSetTest.java @@ -22,8 +22,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.partition.PartitionUtils; import java.util.HashSet; @@ -142,7 +142,7 @@ void testSetParallel(HugeAtomicGrowingBitSet bitSet) { } }, Optional.empty()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); assertThat(bitSet.cardinality()).isEqualTo(nodeCount); } diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java index 2393f1a7a27..e6bbed21816 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java @@ -27,8 +27,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.partition.PartitionUtils; import java.util.Arrays; @@ -184,7 +184,7 @@ void testAddingMultipleNodesInParallel(@ForAll("fixedSizeIds") long[] originalId Optional.of(100) ); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var map = builder.build(); diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java index 0087b7fd110..870a2f449b4 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/AlphaCypherAggregation.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.projection; -import com.neo4j.gds.internal.CustomProceduresUtil; import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.compat.CompatUserAggregationFunction; @@ -103,8 +102,8 @@ public static CompatUserAggregationFunction newInstance() { @Override public CompatUserAggregator create(Context ctx) throws ProcedureException { - var databaseService = CustomProceduresUtil.lookupSafeComponentProvider(ctx, GraphDatabaseService.class); - var username = CustomProceduresUtil.lookupSafeComponentProvider(ctx, Username.class); + var databaseService = Neo4jProxy.lookupComponentProvider(ctx, GraphDatabaseService.class, true); + var username = Neo4jProxy.lookupComponentProvider(ctx, Username.class, true); var runsOnCompositeDatabase = Neo4jProxy.isCompositeDatabase(databaseService); var writeMode = runsOnCompositeDatabase diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java index 9e5f9aea08e..357e80550d1 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.projection; -import com.neo4j.gds.internal.CustomProceduresUtil; import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.compat.CompatUserAggregationFunction; @@ -102,8 +101,8 @@ public static CompatUserAggregationFunction newInstance() { @Override public CompatUserAggregator create(Context ctx) throws ProcedureException { - var databaseService = CustomProceduresUtil.lookupSafeComponentProvider(ctx, GraphDatabaseService.class); - var username = CustomProceduresUtil.lookupSafeComponentProvider(ctx, Username.class); + var databaseService = Neo4jProxy.lookupComponentProvider(ctx, GraphDatabaseService.class, true); + var username = Neo4jProxy.lookupComponentProvider(ctx, Username.class, true); var runsOnCompositeDatabase = Neo4jProxy.isCompositeDatabase(databaseService); var writeMode = runsOnCompositeDatabase diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index 246cb567388..c9e784b96df 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.exceptions.KernelException; import org.neo4j.gds.BaseProcTest; import org.neo4j.gds.BaseTest; import org.neo4j.gds.NodeLabel; @@ -39,6 +40,7 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.catalog.GraphDropProc; import org.neo4j.gds.catalog.GraphListProc; +import org.neo4j.gds.compat.CompatUserAggregationFunction; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.RandomGraphTestCase; @@ -46,6 +48,7 @@ import org.neo4j.gds.extension.Neo4jGraph; import org.neo4j.gds.wcc.WccStreamProc; import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.RelationshipType; import org.neo4j.graphdb.Result; import org.neo4j.kernel.api.procedure.GlobalProcedures; @@ -86,8 +89,7 @@ class CypherAggregationTest extends BaseProcTest { @BeforeEach void setup() throws Exception { - var procedures = GraphDatabaseApiProxy.resolveDependency(db, GlobalProcedures.class); - procedures.register(Neo4jProxy.callableUserAggregationFunction(CypherAggregation.newInstance()), true); + registerUserAggregationFunction(db, CypherAggregation.newInstance()); registerProcedures(GraphDropProc.class, GraphListProc.class, WccStreamProc.class); } @@ -378,6 +380,9 @@ void testNodeLabels(String labels) { "RETURN gds.graph.project('g', s, null, { sourceNodeLabels: " + labels + ", targetNodeLabels: NULL })"); var graphStore = GraphStoreCatalog.get("", db.databaseName(), "g").graphStore(); + + assertThat(graphStore.relationshipTypes()).containsExactlyInAnyOrderElementsOf(graphStore.schema().relationshipSchema().availableTypes()); + assertThat(graphStore.nodeLabels()).extracting(NodeLabel::name).containsExactly("A", "B"); } @@ -1055,4 +1060,16 @@ void testExplicitlyMissingSourceOrTargetNodeInformation(String presentKey, Strin assertThatCode(() -> runQuery(query)) .doesNotThrowAnyException(); } + + private static void registerUserAggregationFunction(GraphDatabaseService db, CompatUserAggregationFunction function) throws + KernelException { + var globalProcedures = GraphDatabaseApiProxy.resolveDependency(db, GlobalProcedures.class); + var alreadyExists = Neo4jProxy.globalProcedureRegistry(globalProcedures).getAllAggregatingFunctions() + .collect(Collectors.toSet()) + .contains(function.signature()); + + if (!alreadyExists) { + GraphDatabaseApiProxy.register(globalProcedures, Neo4jProxy.callableUserAggregationFunction(function)); + } + } } diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java index 89d2f2614f9..066bc66abe5 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java @@ -31,9 +31,9 @@ public enum SyntaxMode { GRAPH_DROP("graph-drop-syntax"), MODEL_DROP("model-drop-syntax"), GRAPH_PROJECT("graph-project-syntax"), - GRAPH_PROJECT_CYPHER("graph-project-cypher-syntax"), + GRAPH_PROJECT_CYPHER("graph-project-cypher-legacy-syntax"), GRAPH_PROJECT_CYPHER_AGGREGATION( - "graph-project-cypher-aggregation-syntax", + "graph-project-cypher-projection-syntax", true, CustomProcedure.Namespace.AGGREGATION_FUNCTION ), @@ -73,8 +73,9 @@ public enum SyntaxMode { SYSTEM_MONITOR("system-monitor-syntax", false), SYS_INFO("debug-sysinfo-syntax", false), WRITE_NODE_LABEL("include-with-write-node-label", false), - MUTATE_NODE_LABEL("include-with-mutate-node-label", false),; - + MUTATE_NODE_LABEL("include-with-mutate-node-label", false), + GRAPH_GENERATE("include-with-graph-generate"),; + private final String mode; public final boolean hasParameters; public final CustomProcedure.Namespace namespace; diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java new file mode 100644 index 00000000000..18b3212ebc4 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc; + +import org.neo4j.gds.beta.generator.GraphGenerateProc; +import org.neo4j.gds.catalog.GraphStreamRelationshipPropertiesProc; +import org.neo4j.gds.catalog.GraphStreamRelationshipsProc; +import org.neo4j.gds.functions.AsNodeFunc; + +import java.util.List; + +class GraphGenerationDocTest extends SingleFileDocTestBase { + + @Override + protected List> functions() { + return List.of(AsNodeFunc.class); + } + + @Override + protected List> procedures() { + return List.of(GraphGenerateProc.class, GraphStreamRelationshipsProc.class, + GraphStreamRelationshipPropertiesProc.class + ); + } + + @Override + protected String adocFile() { + return "pages/management-ops/projections/graph-generation.adoc"; + } +} diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java index 0a920868b5c..769e41fb284 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherAggregationDocTest.java @@ -45,6 +45,6 @@ protected List> procedures() { @Override protected String adocFile() { - return "pages/management-ops/projections/graph-project-cypher-aggregation.adoc"; + return "pages/management-ops/projections/graph-project-cypher-projection.adoc"; } } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java index 020f99e39e3..40fe8c4663c 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/GraphProjectCypherDocTest.java @@ -39,6 +39,6 @@ protected List> procedures() { @Override protected String adocFile() { - return "pages/management-ops/projections/graph-project-cypher.adoc"; + return "pages/management-ops/projections/graph-project-cypher-legacy.adoc"; } } diff --git a/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/AllPairsShortestPathSyntaxTest.java similarity index 63% rename from neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java rename to doc-test/src/test/java/org/neo4j/gds/doc/syntax/AllPairsShortestPathSyntaxTest.java index c07f748570c..a933f8d4e64 100644 --- a/neo4j-api/src/main/java/org/neo4j/gds/api/EmptyDependencyResolver.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/AllPairsShortestPathSyntaxTest.java @@ -17,23 +17,24 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -package org.neo4j.gds.api; +package org.neo4j.gds.doc.syntax; -import org.neo4j.common.DependencyResolver; +import java.util.List; -public final class EmptyDependencyResolver extends DependencyResolver.Adapter { +import static org.neo4j.gds.doc.syntax.SyntaxMode.STREAM; - public static final EmptyDependencyResolver INSTANCE = new EmptyDependencyResolver(); - - private EmptyDependencyResolver() {} +class AllPairsShortestPathSyntaxTest extends SyntaxTestBase { @Override - public T resolveDependency(Class type, SelectionStrategy selector) { - return null; + protected Iterable syntaxModes() { + return List.of( + SyntaxModeMeta.of(STREAM) + ); } @Override - public boolean containsDependency(Class type) { - return false; + protected String adocFile() { + return "pages/alpha-algorithms/all-pairs-shortest-path.adoc"; } + } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java new file mode 100644 index 00000000000..20ca0ed6b04 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc.syntax; + +import java.util.List; + +import static org.neo4j.gds.doc.syntax.SyntaxMode.GRAPH_GENERATE; + +class GraphGenerationSyntaxTest extends SyntaxTestBase { + + @Override + protected Iterable syntaxModes() { + return List.of(SyntaxModeMeta.of(GRAPH_GENERATE)); + } + + @Override + protected String adocFile() { + return "pages/management-ops/projections/graph-generation.adoc"; + } +} diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java index b51b9968bc0..3c60ba825e5 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherAggregationSyntaxTest.java @@ -32,6 +32,6 @@ protected Iterable syntaxModes() { @Override protected String adocFile() { - return "pages/management-ops/projections/graph-project-cypher-aggregation.adoc"; + return "pages/management-ops/projections/graph-project-cypher-projection.adoc"; } } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java index 3314b16b712..c93cb56f32a 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphProjectCypherSyntaxTest.java @@ -32,6 +32,6 @@ protected Iterable syntaxModes() { @Override protected String adocFile() { - return "pages/management-ops/projections/graph-project-cypher.adoc"; + return "pages/management-ops/projections/graph-project-cypher-legacy.adoc"; } } diff --git a/doc/antora.yml b/doc/antora.yml index 9066ee038f2..e8371acbe2e 100644 --- a/doc/antora.yml +++ b/doc/antora.yml @@ -1,6 +1,6 @@ name: graph-data-science title: Neo4j Graph Data Science -version: '2.4-preview' +version: '2.4' start_page: ROOT:index.adoc nav: - modules/ROOT/content-nav.adoc diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 676f5ea2e10..75e849c4e44 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -18,8 +18,8 @@ * xref:management-ops/index.adoc[] ** xref:management-ops/graph-catalog-ops.adoc[] *** xref:management-ops/projections/graph-project.adoc[] -*** xref:management-ops/projections/graph-project-cypher.adoc[] -*** xref:management-ops/projections/graph-project-cypher-aggregation.adoc[] +*** xref:management-ops/projections/graph-project-cypher-legacy.adoc[] +*** xref:management-ops/projections/graph-project-cypher-projection.adoc[] *** xref:graph-project-apache-arrow.adoc[] *** xref:management-ops/projections/graph-project-subgraph.adoc[] *** xref:management-ops/projections/rwr.adoc[] @@ -160,11 +160,13 @@ *** xref:operations-reference/machine-learning-references.adoc[] *** xref:operations-reference/additional-operation-references.adoc[] *** xref:operations-reference/configuration-settings.adoc[] -** xref:appendix-b/index.adoc[] -*** xref:appendix-b/migration-algos-common.adoc[] -*** xref:appendix-b/migration-graph-projection.adoc[] -*** xref:appendix-b/migration-graph-listing.adoc[] -*** xref:appendix-b/migration-graph-drop.adoc[] -*** xref:appendix-b/migration-memory-estimation.adoc[] -*** xref:appendix-b/migration-algorithms.adoc[] -*** xref:appendix-b/migration-ml.adoc[] +** xref:migration-gds-1-to-gds-2/index.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-algos-common.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-graph-projection.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-graph-listing.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-graph-drop.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-memory-estimation.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-algorithms.adoc[] +*** xref:migration-gds-1-to-gds-2/migration-ml.adoc[] +** xref:migration-lcp-to-cpv2/index.adoc[] +** xref:migration-alpha-cp-to-cpv2/index.adoc[] diff --git a/doc/modules/ROOT/pages/algorithms/auxiliary.adoc b/doc/modules/ROOT/pages/algorithms/auxiliary.adoc index 94dd223e9b9..b1fc27debe5 100644 --- a/doc/modules/ROOT/pages/algorithms/auxiliary.adoc +++ b/doc/modules/ROOT/pages/algorithms/auxiliary.adoc @@ -6,9 +6,11 @@ Auxiliary procedures are extra tools that can be useful in your workflow. + The Neo4j GDS library includes the following auxiliary procedures, grouped by quality tier: +* Production-ready +** xref:algorithms/scale-properties.adoc[Scale Properties] + * Beta ** xref:beta-algorithms/collapse-path.adoc[Collapse Path] -** xref:algorithms/scale-properties.adoc[Scale Properties] * Alpha ** xref:alpha-algorithms/one-hot-encoding.adoc#algorithms-one-hot-encoding-sample[One Hot Encoding] diff --git a/doc/modules/ROOT/pages/algorithms/degree-centrality.adoc b/doc/modules/ROOT/pages/algorithms/degree-centrality.adoc index 47a0436e3f7..a4970752c20 100644 --- a/doc/modules/ROOT/pages/algorithms/degree-centrality.adoc +++ b/doc/modules/ROOT/pages/algorithms/degree-centrality.adoc @@ -27,7 +27,7 @@ It can be applied to heterogenous graphs, however the algorithm will not calcula For more information on this algorithm, see: -* http://leonidzhukov.net/hse/2014/socialnetworks/papers/freeman79-centrality.pdf[Linton C. Freeman: Centrality in Social Networks Conceptual Clarification, 1979.^] +* https://www.cin.ufpe.br/%7Erbcp/taia/Freeman1979-centrality.pdf[Linton C. Freeman: Centrality in Social Networks Conceptual Clarification, 1979.^] [[algorithms-degree-centrality-usecase]] diff --git a/doc/modules/ROOT/pages/algorithms/k-core.adoc b/doc/modules/ROOT/pages/algorithms/k-core.adoc index 2b30a5d59f5..d8d54d7ffca 100644 --- a/doc/modules/ROOT/pages/algorithms/k-core.adoc +++ b/doc/modules/ROOT/pages/algorithms/k-core.adoc @@ -5,7 +5,9 @@ :result: core value :algorithm: K-Core Decomposition +:no-directed: :undirected: + include::partial$/algorithms/shared/algorithm-traits.adoc[] [[algorithms-k-core-intro]] diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 400ee19410e..b7f276cd175 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -37,20 +37,19 @@ For more information on this algorithm, see: [[algorithms-kmeans-sampling]] == Initial Centroid Sampling -The algorithm starts by picking `k` centroids by randomly sampling from the set of available nodes. +The algorithm starts by picking `k` centroids by randomly sampling from the set of available nodes. There are two different sampling strategies. Uniform:: -With uniform sampling, each node has the same probability to be picked as one of the `k` initial centroids. +With uniform sampling, each node has the same probability to be picked as one of the `k` initial centroids. This is the default sampler for K-Means denoted with the `uniform` parameter. K-Means++:: -This sampling strategy adapts the well-known -K-means\++ http://ilpubs.stanford.edu:8090/778/1/2006-13.pdf[initialization algorithm] for K-Means. -The sampling begins by choosing the first centroid uniformly at random. -Then, the remaining `k-1` centroids are picked one-by-one based on weighted random sampling. +This sampling strategy adapts the well-known K-means\++ initialization algorithmfootnote:[Arthur, David and Sergei Vassilvitskii. "k-means{plus}{plus}: The Advantages of Careful Seeding." _ACM-SIAM Symposium on Discrete Algorithms_ (2007).] for K-Means. +The sampling begins by choosing the first centroid uniformly at random. +Then, the remaining `k-1` centroids are picked one-by-one based on weighted random sampling. That is, the probability a node is chosen as the next centroid is proportional to its minimum distance from the already picked centroids. Nodes with larger distance hence have higher chance to be picked as a centroid. This sampling strategy tries to spread the initial clusters more evenly so as to obtain a better final clustering. This option can be enabled by choosing `kmeans++` as the initial sampler in the configuration. @@ -105,7 +104,7 @@ include::partial$/algorithms/kmeans/specific-configuration.adoc[] | nodeId | Integer | Node ID. | communityId | Integer | The community ID. | distanceFromCentroid | Float | Distance of the node from the centroid of its community. -| silhouette | Float | Silhouette score of the node. +| silhouette | Float | Silhouette score of the node. |=== ====== diff --git a/doc/modules/ROOT/pages/algorithms/knn.adoc b/doc/modules/ROOT/pages/algorithms/knn.adoc index 5ef0c002e5c..a3915f638e5 100644 --- a/doc/modules/ROOT/pages/algorithms/knn.adoc +++ b/doc/modules/ROOT/pages/algorithms/knn.adoc @@ -680,11 +680,11 @@ ORDER BY similarity DESCENDING, Person1, Person2 [opts="header"] |=== | Person1 | Person2 | similarity -| "Alice" | "Carol" | 0.931216931216931 -| "Carol" | "Alice" | 0.931216931216931 -| "Bob" | "Carol" | 0.432336103416436 -| "Eve" | "Alice" | 0.366920651602733 -| "Dave" | "Bob" | 0.243466706038683 +| "Alice" | "Carol" | 0.887431553441766 +| "Carol" | "Alice" | 0.887431553441766 +| "Bob" | "Carol" | 0.467442948683302 +| "Eve" | "Bob" | 0.37003618661809 +| "Dave" | "Bob" | 0.288711317895105 |=== Note that the two distinct maps in the query could be merged to a single one. diff --git a/doc/modules/ROOT/pages/algorithms/leiden.adoc b/doc/modules/ROOT/pages/algorithms/leiden.adoc index 2bb39ceea9b..c1976b37fe7 100644 --- a/doc/modules/ROOT/pages/algorithms/leiden.adoc +++ b/doc/modules/ROOT/pages/algorithms/leiden.adoc @@ -151,7 +151,7 @@ YIELD modularity: Float, modularities: List of Float, nodeCount: Integer, - didConverge: Integer, + didConverge: Boolean, nodePropertiesWritten: Integer, communityDistribution: Map, configuration: Map @@ -207,7 +207,7 @@ YIELD modularity: Float, modularities: List of Float, nodeCount: Integer, - didConverge: Integer, + didConverge: Boolean, nodePropertiesWritten: Integer, communityDistribution: Map, configuration: Map diff --git a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc index 7189e189d75..2452a266f44 100644 --- a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc +++ b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc @@ -29,7 +29,7 @@ The introduction of a new Pregel algorithm can be separated in two main steps. First, we need to implement the algorithm using the Pregel Java API. Second, we need to expose the algorithm via a Cypher procedure to make use of it. -For an example on how to expose a custom Pregel computation via a Neo4j procedure, have a look at the https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-example/src/main/java/org/neo4j/gds/beta/pregel[Pregel examples]. +For an example on how to expose a custom Pregel computation via a Neo4j procedure, have a look at the https://github.com/neo4j/graph-data-science/tree/2.4/examples/pregel-example/src/main/java/org/neo4j/gds/beta/pregel[Pregel examples]. [[algorithms-pregel-api-java]] == Pregel Java API @@ -549,7 +549,7 @@ For more details, please refer to the xref:algorithms/pregel-api.adoc#algorithms === Building and installing a Neo4j plugin In order to use a Pregel algorithm in Neo4j via a procedure, we need to package it as Neo4j plugin. -The https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-bootstrap[pregel-bootstrap] project is a good starting point. +The https://github.com/neo4j/graph-data-science/tree/2.4/examples/pregel-bootstrap[pregel-bootstrap] project is a good starting point. The `build.gradle` file within the project contains all the dependencies necessary to implement a Pregel algorithm and to generate corresponding procedures. Make sure to change the `gdsVersion` and `neo4jVersion` according to your setup. @@ -578,7 +578,7 @@ dbms.security.procedures.allowlist=custom.pregel.proc.* [[algorithms-pregel-api-example]] == Examples -The https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-example[pregel-examples] module contains a set of examples for Pregel algorithms. +The https://github.com/neo4j/graph-data-science/tree/2.4/examples/pregel-example[pregel-examples] module contains a set of examples for Pregel algorithms. The algorithm implementations demonstrate the usage of the Pregel API. Along with each example, we provide test classes that can be used as a guideline on how to write tests for custom algorithms. To play around, we recommend copying one of the algorithms into the `pregel-bootstrap` project, build it and setup the plugin in Neo4j. diff --git a/doc/modules/ROOT/pages/algorithms/scale-properties.adoc b/doc/modules/ROOT/pages/algorithms/scale-properties.adoc index 9ddb0babde2..e22ab0310dc 100644 --- a/doc/modules/ROOT/pages/algorithms/scale-properties.adoc +++ b/doc/modules/ROOT/pages/algorithms/scale-properties.adoc @@ -1,5 +1,4 @@ [[algorithms-scale-properties]] -[.beta] = Scale Properties :description: This section describes the Scale Properties algorithm in the Neo4j Graph Data Science library. :page-aliases: alpha-algorithms/scale-properties @@ -8,8 +7,6 @@ :entity: node :result: scaled properties -include::partial$/operations-reference/beta-note.adoc[] - [[algorithms-scale-properties-intro]] == Introduction diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index 814cc203f00..db4a95364bc 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -521,7 +521,7 @@ CALL gds.graph.project( _Step 4_ -[role=query-example, group=seeding] +[role=query-example, group=seeding, no-result=true] -- .The following will run the algorithm in `stream` mode using `seedProperty`: [source, cypher, role=noplay] diff --git a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc index a93754049b9..7f6ca93a25c 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc @@ -43,6 +43,8 @@ Plain cypher does not support filtering `Infinity` values, so `gds.util.isFinite [[algorithm-all-pairs-shortest-path-syntax]] == Syntax +[.include-with-stream] +====== .The following will run the algorithm and stream results: [source, cypher, role=noplay] ---- @@ -50,18 +52,30 @@ CALL gds.alpha.allShortestPaths.stream( graphName: string, configuration: map ) -YIELD startNodeId, targetNodeId, distance +YIELD sourceNodeId, targetNodeId, distance ---- -.Parameters -[opts="header",cols="1,1,1,1,4"] +include::partial$/algorithms/common-configuration/common-parameters.adoc[] + +.Configuration +[opts="header",cols="3,2,3m,2,8"] |=== -| Name | Type | Default | Optional | Description +| Name | Type | Default | Optional | Description +include::partial$/algorithms/common-configuration/common-stream-stats-configuration-entries.adoc[] | xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. -| concurrency | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. Also provides the default value for 'readConcurrency' and 'writeConcurrency'. This is dependent on the Neo4j edition; for more information, see xref:installation/System-requirements.adoc#system-requirements-cpu[CPU]. -| readConcurrency | Integer | value of 'concurrency' | yes | The number of concurrent threads used for reading the graph. + +|=== + +.Results +[opts="header",cols="1,1,6"] +|=== +| Name | Type | Description +| sourceNodeId | Integer | The source node. +| targetNodeId | Integer | The target node. +| distance | Float | The distance of the shortest path from source to target. |=== +====== [[algorithm-all-pairs-shortest-path-sample]] == All Pairs Shortest Path algorithm sample @@ -147,9 +161,9 @@ LIMIT 10 This query returned the top 10 pairs of nodes that are the furthest away from each other. F and E appear to be quite distant from the others. -=== Using Cypher aggregation +=== Using Cypher projection -.The following will project and store an undirected graph using cypher aggregation: +.The following will project and store an undirected graph using cypher projection: [source, cypher, role=noplay graph-project-query, group=cypher] ---- MATCH (src:Loc)-[r:ROAD]->(trg:Loc) diff --git a/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc b/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc index fbef5767d97..03ab54dcdf9 100644 --- a/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc +++ b/doc/modules/ROOT/pages/common-usage/memory-estimation.adoc @@ -82,8 +82,8 @@ With this it is possible to measure the memory consumption of projecting a graph | Name | Type | Default | Optional | Description | xref:management-ops/projections/graph-project.adoc#node-projection-syntax[node projection] | String, List of String or Map | null | yes | The node projection used for anonymous graph creation via a Native projection. | xref:management-ops/projections/graph-project.adoc#relationship-projection-syntax[relationship projection] | String, List of String or Map | null | yes | The relationship projection used for anonymous graph creation a Native projection. -| nodeQuery | String | null | yes | The Cypher query used to select the nodes for anonymous graph creation via a Cypher projection. -| relationshipQuery | String | null | yes | The Cypher query used to select the relationships for anonymous graph creation via a Cypher projection. +| nodeQuery | String | null | yes | The Cypher query used to select the nodes for anonymous graph creation via a Legacy Cypher projection. +| relationshipQuery | String | null | yes | The Cypher query used to select the relationships for anonymous graph creation via a Legacy Cypher projection. | nodeProperties | String, List of String or Map | null | yes | The node properties to project during anonymous graph creation. | relationshipProperties | String, List of String or Map | null | yes | The relationship properties to project during anonymous graph creation. | xref:common-usage/running-algos.adoc#common-configuration-concurrency[concurrency] | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. Also provides the default value for 'readConcurrency' and 'writeConcurrency'. @@ -116,6 +116,7 @@ The `nodeProjection` and `relationshipProjection` parameters follow the same syn The result of running `gds.graph.project.estimate` has the same form as the algorithm memory estimation results above. +[[estimate-procedure-fictive-graph]] It is also possible to estimate the memory of a fictive graph, by explicitly specifying its node and relationship count. Using this feature, one can estimate the memory consumption of an arbitrarily sized graph. @@ -153,7 +154,7 @@ YIELD requiredMemory, treeView, mapView, bytesMin, bytesMax, nodeCount, relation | "593 KiB" | 607576 | 607576 | 100 | 1000 |=== -The xref:management-ops/projections/graph-project-cypher.adoc[`gds.graph.project.cypher`] procedure has to execute both, the `nodeQuery` and `relationshipQuery`, in order to count the number of nodes and relationships of the graph. +The xref:management-ops/projections/graph-project-cypher-legacy.adoc[`gds.graph.project.cypher`] procedure has to execute both, the `nodeQuery` and `relationshipQuery`, in order to count the number of nodes and relationships of the graph. .Syntax [source, cypher, role=noplay] diff --git a/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc b/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc index eb9603ddb33..a6f763b7a50 100644 --- a/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc +++ b/doc/modules/ROOT/pages/graph-catalog-apache-arrow-ops.adoc @@ -47,7 +47,8 @@ To stream a single node property, the client needs to encode that information in procedure_name: "gds.graph.nodeProperty.stream", configuration: { node_labels: ["*"], - node_property: "foo" + node_property: "foo", + list_node_labels: true } } ---- @@ -61,20 +62,22 @@ The specific configuration needs to include the following keys: | Name | Type | Description | node_labels | String or List of Strings | Stream only properties for nodes with the given labels. | node_property | String | The node property in the graph to stream. +| list_node_labels | Boolean | Whether to include node labels of the respective nodes in the result. |=== The schema of the result records is identical to the corresponding procedure: .Results -[opts="header",cols="2,3,5"] +[opts="header",cols="2,3,5, 1"] |=== -| Name | Type | Description -|nodeId | Integer | The id of the node. +| Name | Type | Description | Optional +| nodeId | Integer | The id of the node. | false .^|propertyValue a| * Integer * Float * List of Integer -* List of Float .^| The stored property value. +* List of Float .^| The stored property value. | false +| labels | List of Strings | The labels of the node. If the `list_node_labels` option was set | true |=== @@ -89,7 +92,8 @@ To stream multiple node properties, the client needs to encode that information procedure_name: "gds.graph.streamNodeProperties", configuration: { node_labels: ["*"], - node_properties: ["foo", "bar", "baz"] + node_properties: ["foo", "bar", "baz"], + list_node_labels: true } } ---- @@ -103,6 +107,7 @@ The specific configuration needs to include the following keys: | Name | Type | Description | node_labels | String or List of Strings | Stream only properties for nodes with the given labels. | node_properties | String or List of Strings | The node properties in the graph to stream. +| list_node_labels | Boolean | Whether to include node labels of the respective nodes in the result. |=== Note that the schema of the result records is not identical to the corresponding procedure. @@ -215,7 +220,7 @@ To stream the topology of one or more relationship types, the client needs to en { graph_name: "my_graph", database_name: "database_name", - procedure_name: "gds.graph.relationshipProperties.stream", + procedure_name: "gds.beta.graph.relationships.stream", configuration: { relationship_types: "REL" } @@ -244,3 +249,58 @@ The schema of the result records is identical to the corresponding procedure: Note, that the relationship type column stores the relationship type encoded as an integer. The corresponding string value needs to be retrieved from the corresponding dictionary value vector. That vector can be loaded from the dictionary provider using the encoding id of the type field. + + +== Partitioning the data streams + +Some use-cases require the data streams to be partitioned. +For example, if the data streams are consumed by a distributed system, the data streams need to be evenly distributed to the members of the distributed system. +To support this use-case, the client can request the data streams to be partitioned by sending the stream request to the `FlightInfo` endpoint of the GDS Flight Server. +The server will then return a number of endpoints, where each endpoint and it's accompanying ticket can be used to stream a partition of the data. +The `concurrency` settings of the ticket can be used to control the number of partitions. + +For example, to stream the topology of one or more relationship types, the client needs to encode that information in the ticket as follows: + +---- +{ + graph_name: "my_graph", + database_name: "database_name", + procedure_name: "gds.beta.graph.relationships.stream", + concurrency: 2, + configuration: { + relationship_types: "REL" + } +} +---- + +This will create at most 2 partitions of the data streams. +The server will answer with 2 tickets: + +---- +[ + { + graph_name: "my_graph", + database_name: "database_name", + procedure_name: "gds.beta.graph.relationships.stream", + concurrency: 4, + partition_offset: 0, + partition_size: 100, + configuration: { + relationship_types: "REL" + } + }, + { + graph_name: "my_graph", + database_name: "database_name", + procedure_name: "gds.beta.graph.relationships.stream", + partition_offset: 100, + partition_size: 100, + concurrency: 4, + configuration: { + relationship_types: "REL" + } + } +] +---- + +Each of the tickets can now be used to request a partition data via the `GET` endpoint of the GDS Flight Server. diff --git a/doc/modules/ROOT/pages/graph-catalog-node-ops.adoc b/doc/modules/ROOT/pages/graph-catalog-node-ops.adoc index b8cf00fca72..267f4130765 100644 --- a/doc/modules/ROOT/pages/graph-catalog-node-ops.adoc +++ b/doc/modules/ROOT/pages/graph-catalog-node-ops.adoc @@ -35,7 +35,8 @@ CALL gds.graph.nodeProperty.stream( ) YIELD nodeId: Integer, - propertyValue: Integer or Float or List of Integer or List of Float + propertyValue: Integer or Float or List of Integer or List of Float, + nodeLabels: List of Strings ---- .Parameters @@ -53,6 +54,7 @@ YIELD |=== | Name | Type | Default | Description | concurrency | Integer | 4 | The number of concurrent threads. Note, this procedure is always running single-threaded. +| listNodeLabels | Boolean | false | Whether to return a list of node labels for each node. |=== .Results @@ -65,6 +67,7 @@ YIELD * Float * List of Integer * List of Float .^| The stored property value. +| nodeLabels | List of Strings | The node labels of the node. |=== ====== @@ -81,7 +84,8 @@ CALL gds.graph.nodeProperties.stream( YIELD nodeId: Integer, nodeProperty: String, - propertyValue: Integer or Float or List of Integer or List of Float + propertyValue: Integer or Float or List of Integer or List of Float, + nodeLabels: List of Strings ---- .Parameters @@ -99,6 +103,7 @@ YIELD |=== | Name | Type | Default | Description | concurrency | Integer | 4 | The number of concurrent threads. Note, this procedure is always running single-threaded. +| listNodeLabels | Boolean | false | Whether to return a list of node labels for each node. |=== .Results @@ -112,6 +117,7 @@ YIELD * Float * List of Integer * List of Float .^| The stored property value. +| nodeLabels | List of Strings | The node labels of the node. |=== ====== @@ -438,6 +444,43 @@ When streaming multiple node properties, the name of each property is included i This adds with some overhead, as each property name must be repeated for each node in the result, but is necessary in order to distinguish properties. ==== + +[[catalog-graph-stream-node-properties-with-labels-example]] + +Additionally, when streaming one or more node properties, we can also return the node labels for each individual node by setting the `listNodeLabels` configuration option. + +[role=query-example] +-- +.Stream multiple node properties with labels: +[source, cypher, role=noplay] +---- +CALL gds.graph.nodeProperties.stream( + 'socialGraph', + ['score'], + ['*'], + { listNodeLabels: true } +) +YIELD nodeId, nodeProperty, propertyValue, nodeLabels +RETURN + gds.util.asNode(nodeId).name AS name, + nodeProperty, + propertyValue, + nodeLabels +---- + +.Results +[opts="header"] +|=== +| name | nodeProperty | propertyValue | nodeLabels +| "Florentin" | "score" | 2.0 | [Person] +| "Adam" | "score" | 1.0 | [Person] +| "Veselin" | "score" | 0.0 | [Person] +| "The Hobbit" | "score" | 0.0 | [Book] +|=== +-- + + + [[catalog-graph-write-node-properties-example]] === Write diff --git a/doc/modules/ROOT/pages/graph-list.adoc b/doc/modules/ROOT/pages/graph-list.adoc index a78de46433c..3f16115903b 100644 --- a/doc/modules/ROOT/pages/graph-list.adoc +++ b/doc/modules/ROOT/pages/graph-list.adoc @@ -95,7 +95,7 @@ CREATE (florentin)-[:KNOWS { since: 2018 }]->(veselin) ---- -Additionally, we will project a few graphs to the graph catalog, for more details see xref:management-ops/projections/graph-project.adoc[native projections] and xref:management-ops/projections/graph-project-cypher.adoc[Cypher projections]. +Additionally, we will project a few graphs to the graph catalog, for more details see xref:management-ops/projections/graph-project.adoc[native projections] and xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projections]. .Project `Person` nodes and `KNOWS` relationships using native projections: [source, cypher, role=noplay graph-project-query] diff --git a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc index 92005eff4bb..024e8876e46 100644 --- a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc +++ b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc @@ -147,13 +147,6 @@ The server acknowledges the action by returning a JSON document including the na } ---- -[NOTE] -==== -Node identifiers are represented by long values in the range of 0 to 2^63. -If the input node id space is sparse and contains very large node id values, one might observe a high memory footprint for the projected graph. -In these situations, the memory footprint of the graph could be reduced by switching to another xref:production-deployment/feature-toggles.adoc#sharded-id-map-feature-toggle[id map implementation]. -==== - [[arrow-send-relationships]] == Sending relationship records via PUT as a Flight stream diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc index 042b05a9329..4f9cce6c48b 100644 --- a/doc/modules/ROOT/pages/index.adoc +++ b/doc/modules/ROOT/pages/index.adoc @@ -32,7 +32,8 @@ The manual covers the following areas: * xref:production-deployment/index.adoc[Production deployment] -- This chapter explains advanced details with regards to common Neo4j components. * xref:python-client/index.adoc[Python client] -- Documentation of the Graph Data Science client for Python users. * xref:operations-reference/appendix-a.adoc[Operations reference] -- Reference of all procedures contained in the Neo4j Graph Data Science library. -* xref:appendix-b/index.adoc[Migration from Graph Data Science library Version 1.x] -- Additional resources - migration guide, books, etc - to help using the Neo4j Graph Data Science library. +* xref:migration-gds-1-to-gds-2/index.adoc[Migration from Graph Data Science library Version 1.x] -- Additional resources - migration guide, books, etc - to help using the Neo4j Graph Data Science library. +* xref:migration-lcp-to-cpv2/index.adoc[Migration from Legacy to new Cypher projections] -- Migration guide to help migration from the Legacy Cypher projections to the new Cypher projections. The source code of the library is available at https://github.com/neo4j/graph-data-science[GitHub]. If you have a suggestion on how we can improve the library or want to report a problem, you can create a https://github.com/neo4j/graph-data-science/issues/new[new issue]. diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index f2243501451..52740a3b587 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -60,9 +60,9 @@ https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#c However, setting a minimum page cache size is still important when projecting graphs: * For xref:management-ops/projections/graph-project.adoc[native projections], the minimum page cache size for projecting a graph can be roughly estimated by `8KB * 100 * readConcurrency`. -* For xref:management-ops/projections/graph-project-cypher.adoc[Cypher projections], a higher page cache is required depending on the query complexity. -* For xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher aggregation], a higher page cache is required depending on the query complexity. +* For xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projections], a higher page cache is required depending on the query complexity. * For xref:graph-project-apache-arrow.adoc[projections through Apache Arrow], the page cache is irrelevant. +* For xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projections], a higher page cache is required depending on the query complexity. However, if it is required to write algorithm results back to Neo4j, the write performance is highly depended on store fragmentation as well as the number of properties and relationships to write. We recommend starting with a page cache size of roughly `250MB * writeConcurrency` and evaluate write performance and adapt accordingly. @@ -74,11 +74,11 @@ Decreasing the page cache size in favor of heap size is *not* recommended if the See https://neo4j.com/docs/operations-manual/4.4/performance/memory-configuration/[Neo4j memory configuration] for general information about page cache sizing. ==== -=== Off-heap memory +=== Native memory -The off-heap space is used by the xref:installation/configure-apache-arrow-server.adoc[Apache Arrow server] to store received data. +Native memory is used by the xref:installation/configure-apache-arrow-server.adoc[Apache Arrow server] to store received data. -If you have the xref:installation/configure-apache-arrow-server.adoc[Apache Arrow server] enabled, we also recommend to reserve some off-heap memory. +If you have the xref:installation/configure-apache-arrow-server.adoc[Apache Arrow server] enabled, we also recommend to reserve some native memory. The amount of memory required depends on the batch size used by the client. Data received through Arrow is temporarily stored in direct memory before being converted and loaded into an on-heap graph. diff --git a/doc/modules/ROOT/pages/installation/index.adoc b/doc/modules/ROOT/pages/installation/index.adoc index 04fe1483b4f..cd1d6915547 100644 --- a/doc/modules/ROOT/pages/installation/index.adoc +++ b/doc/modules/ROOT/pages/installation/index.adoc @@ -65,9 +65,18 @@ CALL gds.debug.sysInfo(); In order to make use of certain features of the GDS library, additional configuration may be necessary. For example, exporting graphs xref:graph-catalog-export-ops.adoc#catalog-graph-export-csv[to CSV files] requires the configuration parameter `gds.export.location` to be set to the folder in which exported graphs are to be stored. -You can find the list of all the configuration options xref:production-deployment/configuration-settings/[here]. +You can find the list of all the configuration options xref:production-deployment/configuration-settings.adoc[here]. Refer to the <<_installation_methods,installation methods>> for details on how to edit a Neo4j database configuration depending on the Neo4j deployment. +== Graph Data Science on MacOS + +If you are running MacOS (x86 or ARM), it is currently required to add the following configuration entry to `neo4j.conf`: + +---- +server.jvm.additional=-Djol.skipHotspotSAAttach=true +---- + + == Reference * xref:installation/supported-neo4j-versions.adoc[Supported Neo4j versions] diff --git a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc index 9d45b14d9ee..524ae107c18 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc @@ -28,4 +28,10 @@ If the procedure allowlist is configured, make sure to also include procedures f ---- dbms.security.procedures.allowlist=gds.* ----- \ No newline at end of file +---- + +If you are running MacOS (x86 or ARM), it is required to add the following configuration entry to the configuration: + +---- +server.jvm.additional=-Djol.skipHotspotSAAttach=true +---- diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index 65ea46ed686..47e81ff9afd 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -15,11 +15,12 @@ dbms.security.procedures.unrestricted=gds.* This configuration entry is necessary because the GDS library accesses low-level components of Neo4j to maximise performance. + -4. Check if the procedure allowlist is enabled in the `$NEO4J_HOME/conf/neo4j.conf` file and add the GDS library if necessary: +4. Check if the procedure allowlist is enabled in the `$NEO4J_HOME/conf/neo4j.conf` file, namely if the `dbms.security.procedures.allowlist` option is _not_ commented out with a leading `#` sign. In this case, add the GDS library to the allowlist: + ---- dbms.security.procedures.allowlist=gds.* ---- +You can find more information on allow listing in the link:https://neo4j.com/docs/operations-manual/current/security/securing-extensions/#allow-listing[Operations Manual]. + 5. Restart the Neo4j Server. diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 203c8963c76..ffe2e38db48 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,13 +10,17 @@ If your version of GDS or Neo4j is not listed in the matrix, you should upgrade. [opts=header] |=== | Neo4j version | Neo4j Graph Data Science -| `5.8` | `2.4`, `2.3.6` or later -| `5.7` | `2.4`, `2.3.3` or later -| `5.6` | `2.4`, `2.3.2` or later -| `5.5` | `2.4`, `2.3.1` or later -| `5.4` | `2.4`, `2.3` -| `5.3` | `2.4`, `2.3` -| `5.2` | `2.4`, `2.3` -| `5.1` | `2.4`, `2.3` -| `4.4.9` or later | `2.4`, `2.3` +| `5.12` | `2.4.6` or later +| `5.11` | `2.4.4` or later +| `5.10` | `2.4.2` or later +| `5.9` | `2.4`, `2.3.9` or later footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] +| `5.8` | `2.4`, `2.3.6` or later footnote:eol[] +| `5.7` | `2.4`, `2.3.3` or later footnote:eol[] +| `5.6` | `2.4`, `2.3.2` or later footnote:eol[] +| `5.5` | `2.4`, `2.3.1` or later footnote:eol[] +| `5.4` | `2.4`, `2.3` footnote:eol[] +| `5.3` | `2.4`, `2.3` footnote:eol[] +| `5.2` | `2.4`, `2.3` footnote:eol[] +| `5.1` | `2.4`, `2.3` footnote:eol[] +| `4.4.9` or later | `2.4`, `2.3` footnote:eol[] |=== diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc index c4c32353c84..2bfab3ba6e1 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/graph-sage.adoc @@ -540,7 +540,10 @@ CALL gds.graph.project( We can now run GraphSAGE in multi-label mode on that graph by specifying the `projectedFeatureDimension` parameter. Multi-label GraphSAGE removes the requirement, that each node in the in-memory graph must have all `featureProperties`. However, the projections are independent per label and even if two labels have the same `featureProperty` they are considered as different features before projection. -The `projectedFeatureDimension` equals the maximum length of the feature-array, i.e., `age` and `cost` both are scalar features plus the list feature `heightAndWeight` which has a length of two. +The `projectedFeatureDimension` should equal the maximum length of the feature-array. +In our example, persons have `age` (1) and `heightAndWeight` (2), summing up to a length of 3. +Instruments only have `cost` with length of 1. +Thus, the `projectedFeatureDimension` should be set to 3. For each node its unique labels properties is projected using a label specific projection to vector space of dimension `projectedFeatureDimension`. Note that the `cost` feature is only defined for the instrument nodes, while `age` and `heightAndWeight` are only defined for persons. @@ -551,7 +554,7 @@ CALL gds.beta.graphSage.train( { modelName: 'multiLabelModel', featureProperties: ['age', 'heightAndWeight', 'cost'], - projectedFeatureDimension: 4 + projectedFeatureDimension: 3 } ) ---- diff --git a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc index 56afdf9793d..017b0e807eb 100644 --- a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc +++ b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc @@ -44,9 +44,10 @@ YIELD .Configuration [opts="header",cols="1,1,1,4"] |=== -| Name | Type | Default | Description -| concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. -| includeGraphs | Boolean | true | Flag to decide whether only models or also graphs should be backed up. +| Name | Type | Default | Description +| concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. +| includeGraphs | Boolean | true | Flag to decide whether only models or also graphs should be backed up. +| useLabelMapping | Boolean | true | Flag to decide whether to use node label mapping when exporting the graph. See xref:graph-catalog-export-ops.adoc#node_label_mapping[exporting graphs to csv] for details. |=== .Results diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index 3e182fea6b7..c699bfcddf3 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -40,8 +40,8 @@ Read more about that in xref:management-ops/administration.adoc[]. Named graphs can be projected from a Neo4j database by using either of - a xref:management-ops/projections/graph-project.adoc[Native projection] -- a xref:management-ops/projections/graph-project-cypher.adoc[Cypher projection] -- or a xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] +- a xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projection] +- or a xref:management-ops/projections/graph-project-cypher-legacy.adoc[Legacy Cypher projection] But graphs can also be projected into the graph catalog from other sources. xref:management-ops/projections/graph-project-subgraph.adoc[Subgraph] and xref:management-ops/projections/rwr.adoc[Graph Sampling] projections allow projecting a new graph based off of an existing graph projection. @@ -52,12 +52,12 @@ xref:management-ops/projections/graph-generation.adoc[Randomised graphs can be g |=== | Name | Description | xref:management-ops/projections/graph-project.adoc[gds.graph.project] | Adds a graph to the catalog using Native projection. -| xref:management-ops/projections/graph-project-cypher.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Cypher projection. -| xref:management-ops/projections/graph-project-cypher-aggregation.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher Aggregation. +| xref:management-ops/projections/graph-project-cypher-projection.adoc[gds.graph.project] | Adds a graph to the catalog using Cypher projection. | xref:management-ops/projections/graph-project-subgraph.adoc[gds.beta.graph.project.subgraph] | Adds a graph to the catalog by filtering an existing graph using node and relationship predicates. | xref:management-ops/projections/rwr.adoc[gds.graph.sample.rwr] | Adds a graph to the catalog by sampling an existing graph using random walk with restarts. | xref:management-ops/projections/cnarw.adoc[gds.graph.sample.cnarw] | Adds a graph to the catalog by sampling an existing graph using Common Neighbour Aware Random Walk algorithm. | xref:management-ops/projections/graph-generation.adoc[gds.beta.graph.generate] | Creates a new random graph projection of the user-defined properties and dimensions. +| xref:management-ops/projections/graph-project-cypher-legacy.adoc[gds.graph.project.cypher] | Adds a graph to the catalog using Legacy Cypher projection. |=== diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc index c484061a1c8..8c1b7d2d7c3 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc @@ -26,13 +26,16 @@ The graph generation is parameterized by three dimensions: [[graph-generation-syntax]] == Syntax - -.The following describes the API for running the algorithm +[.include-with-graph-generate] +==== +.The following describes the API for running the graph generation procedure [source, cypher, role=noplay] ---- -CALL gds.beta.graph.generate(graphName: String, nodeCount: Integer, averageDegree: Integer, { - relationshipDistribution: String, - relationshipProperty: Map +CALL gds.beta.graph.generate( + graphName: String, + nodeCount: Integer, + averageDegree: Integer, + configuration: Map }) YIELD name, nodes, relationships, generateMillis, relationshipSeed, averageDegree, relationshipDistribution, relationshipProperty ---- @@ -72,6 +75,7 @@ YIELD name, nodes, relationships, generateMillis, relationshipSeed, averageDegre | relationshipDistribution | String | The probability distribution method used to connect generated nodes. | relationshipProperty | String | The configuration of the generated relationship property. |=== +==== [[graph-generation-distribution]] == Relationship Distribution @@ -112,3 +116,115 @@ Currently, there are two supported methods to generate relationship properties: * `FIXED` - Assigns a fixed value to every relationship. The `value` parameter must be set. * `RANDOM` - Assigns a random value between the lower (`min`) and upper (`max`) bound. + +[[graph-generation-example]] +== Examples + +In the following we will demonstrate the usage of the random graph generation procedure. + +[[graph-generation-unweighted]] +=== Generating unweighted graphs + +[role=query-example,group=unweighted] +-- +.The following will produce a graph with unweighted relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.generate('graph',5,2, {relationshipSeed:19}) +YIELD name, nodes, relationships, relationshipDistribution +---- + +.Results +[opts="header"] +|=== +| name | nodes | relationships | relationshipDistribution +| "graph"| 5 | 10 | "UNIFORM" +|=== +-- + +A new in-memory graph called `graph` with `5` nodes and `10` relationships has been created and added to the graph catalog. +We can examine its topology with the `gds.beta.graph.relationships` procedure. + +[role=query-example,group=unweighted] +-- +.The following will show the produced relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.relationships.stream('graph') +YIELD sourceNodeId,targetNodeId +RETURN sourceNodeId as source, targetNodeId as target +ORDER BY source ASC,target ASC +---- + +.Results +[opts="header"] +|=== +| source |target +| 0 | 1 +| 0 | 2 +| 1 | 0 +| 1 | 4 +| 2 | 1 +| 2 | 4 +| 3 | 0 +| 3 | 1 +| 4 | 0 +| 4 | 3 +|=== +-- + +[[graph-generation-weighted]] +=== Generating weighted graphs + +To generated graphs with weighted relationships we must specify the `relationshipProperty` parameter as discussed xref:graph-generation-relationship-property[above]. + +[role=query-example,group=weighted] +-- +.The following will produce a graph with weighted relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.generate('weightedGraph',5,2, {relationshipSeed:19, + relationshipProperty: {type: 'RANDOM', min: 5.0, max: 10.0, name: 'score'}}) +YIELD name, nodes, relationships, relationshipDistribution +---- + +.Results +[opts="header"] +|=== +|name| nodes | relationships | relationshipDistribution +| "weightedGraph"| 5 | 10 | "UNIFORM" +|=== +-- + +The produced graph, `weightedGraph`, has a property named `score` containing a random value between 5.0 and 10.0 for each relationship. +We can use `gds.graph.relationshipProperty.stream` to stream the relationships of the graph along with their score values. + +[role=query-example,group=weighted] +-- +.The following will show the produced relationships +[source,cypher,role=noplay] +---- +CALL gds.graph.relationshipProperty.stream('weightedGraph','score') +YIELD sourceNodeId, targetNodeId, propertyValue +RETURN sourceNodeId as source, targetNodeId as target, propertyValue as score +ORDER BY source ASC,target ASC, score +---- + +.Results +[opts="header"] +|=== +| source |target | score +| 0 | 1 | 6.258381821615686 +| 0 | 2 | 6.791408433596591 +| 1 | 4 | 8.747179900968224 +| 1 | 4 | 9.469695236791349 +| 2 | 0 | 7.061710127800056 +| 2 | 1 | 5.060444167785128 +| 3 | 1 | 6.308266834622538 +| 3 | 4 | 9.040323743901354 +| 4 | 1 | 7.939688205556302 +| 4 | 1 | 7.988277646384441 +|=== +-- + +Notice that despite `graph` and `weightedGraph` having the same `relationshipSeed`, their actual topology differs. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc similarity index 89% rename from doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc rename to doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc index afad607fb42..a89a7c234d3 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-legacy.adoc @@ -1,9 +1,17 @@ -[[catalog-graph-project-cypher]] -= Projecting graphs using Cypher -:description: This section details projecting GDS graphs using `Cypher` projections. +[[catalog-graph-project-cypher-legacy]] += Projecting graphs using Cypher (deprecated) +:description: This section details projecting GDS graphs using legacy `Cypher` projections. +:page-aliases: managments-ops/projections/graph-project-cypher.adoc -Using Cypher projections is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. -Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). +[WARNING] +-- +This page describes the Legacy Cypher projection, which is deprecated. +The replacement is to use the new Cypher projection, which is described in xref:management-ops/projections/graph-project-cypher-projection.adoc[Projecting graphs using Cypher]. +A migration guide is available at xref:migration-lcp-to-cpv2/index.adoc[Appendix C, Migration from Legacy to new Cypher projection]. +-- + +Legacy Cypher projections are a more flexible and expressive approach compared to xref:management-ops/projections/graph-project.adoc[native projections]. +A Legacy Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. == Considerations @@ -22,18 +30,18 @@ The projected graphs will reside in the catalog until either: === Node property support -Cypher projections can only project a limited set of node property types from a Cypher query. +Legacy Cypher projections can only project a limited set of node property types from a Cypher query. The xref:management-ops/node-properties.adoc#node-properties-supported[Node Properties page] details which node property types are supported. -Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Cypher projection. +Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Legacy Cypher projection. -[[graph-project-cypher-syntax]] +[[graph-project-cypher-legacy-syntax]] == Syntax -A Cypher projection takes three mandatory arguments: `graphName`, `nodeQuery` and `relationshipQuery`. +A Legacy Cypher projection takes three mandatory arguments: `graphName`, `nodeQuery` and `relationshipQuery`. In addition, the optional `configuration` parameter allows us to further configure graph creation. -[.graph-project-cypher-syntax] +[.graph-project-cypher-legacy-syntax] -- [source, cypher, role=noplay] ---- @@ -58,7 +66,7 @@ CALL gds.graph.project.cypher( | graphName | no | The name under which the graph is stored in the catalog. | nodeQuery | no | Cypher query to project nodes. The query result must contain an `id` column. Optionally, a `labels` column can be specified to represent node labels. Additional columns are interpreted as properties. | relationshipQuery | no | Cypher query to project relationships. The query result must contain `source` and `target` columns. Optionally, a `type` column can be specified to represent relationship type. Additional columns are interpreted as properties. -| configuration | yes | Additional parameters to configure the Cypher projection. +| configuration | yes | Additional parameters to configure the Legacy Cypher projection. |=== .Configuration @@ -173,15 +181,15 @@ YIELD === Relationship orientation The native projection supports specifying an orientation per relationship type. -The Cypher projection treats every relationship returned by the relationship query as if it were in `NATURAL` orientation and creates a directed relationship from the first provided id (source) to the second (target). +The Legacy Cypher projection treats every relationship returned by the relationship query as if it were in `NATURAL` orientation and creates a directed relationship from the first provided id (source) to the second (target). Projecting in `REVERSE` orientation can be achieved by switching the order of ids in the RETURN clause such as `MATCH (n)-[r:KNOWS]->(m) RETURN id(m) AS source, id(n) AS target, type(r) AS type`. -It not possible to project graphs in `UNDIRECTED` orientation when Cypher projections are used. +It not possible to project graphs in `UNDIRECTED` orientation when Legacy Cypher projections are used. [NOTE] -- Some algorithms require that the graph was loaded with `UNDIRECTED` orientation. -These algorithms can not be used with a graph projected by a Cypher projection. +These algorithms can not be used with a graph projected by a Legacy Cypher projection. -- @@ -222,7 +230,7 @@ RETURN graphName, nodes, rels -- The projected `graphWithProperties` graph contains five nodes and six relationships. -In a Cypher projection every node from the `nodeQuery` gets the same node properties, which means you can't have label-specific properties. +In a Legacy Cypher projection every node from the `nodeQuery` gets the same node properties, which means you can't have label-specific properties. For instance in the example above the `Person` nodes will also get `ratings` and `price` properties, while `Book` nodes get the `age` property. Further, the `price` property has a default value of `5.0`. @@ -306,7 +314,7 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-aggregation.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. +We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. === Parallel relationships @@ -481,7 +489,7 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -If we compare the results to the ones from xref:management-ops/projections/graph-project-cypher.adoc#cypher-relationship-properties[Relationship properties], we can see that using `IS NOT NULL` is filtering out the relationship from Veselin to the book Frankenstein. +If we compare the results to the ones from xref:management-ops/projections/graph-project-cypher-legacy.adoc#cypher-relationship-properties[Relationship properties], we can see that using `IS NOT NULL` is filtering out the relationship from Veselin to the book Frankenstein. This functionality is only expressible with xref:management-ops/projections/graph-project.adoc[native projections] by projecting a xref:management-ops/projections/graph-project-subgraph.adoc[subgraph]. [[cypher-projection-parameters]] diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc similarity index 90% rename from doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc rename to doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc index d14584dd6c8..d4160a58833 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-projection.adoc @@ -1,10 +1,11 @@ -[[catalog-graph-project-cypher-aggregation]] -= Projecting graphs using Cypher Aggregation -:description: This section details projecting GDS graphs using `Cypher` aggregations. +[[catalog-graph-project-cypher-projection]] += Projecting graphs using Cypher +:description: This section details projecting GDS graphs using `Cypher` projections. +:page-aliases: management-ops/projections/graph-project-cypher-aggregation -Using Cypher aggregations is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. -Cypher aggregations are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). +Cypher projections are a more flexible and expressive approach compared to xref:management-ops/projections/graph-project.adoc[native projections]. +A Cypher projection uses Cypher to create (_project_) an in-memory graph from the Neo4j database. == Considerations @@ -23,9 +24,9 @@ The projected graphs will reside in the catalog until either: === Node property support -Cypher aggregations can only project a limited set of node property types from a Cypher query. +Cypher projections can only project a limited set of node property types from a Cypher query. The xref:management-ops/node-properties.adoc#node-properties-supported[Node Properties page] details which node property types are supported. -Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Cypher aggregation. +Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Cypher projection. === Selection of node properties and labels @@ -34,17 +35,17 @@ This is important when a node can be a source node as well as a target node and Relevant configuration options are `sourceNodeProperties`, `targetNodeProperties`, `sourceNodeLabels` and `targetNodeLabels`. -[[graph-project-cypher-aggregation-syntax]] +[[graph-project-cypher-projection-syntax]] == Syntax -A Cypher aggregation is used in a query as an aggregation over the relationships that are being projected. +A Cypher projection is used in a query as an aggregation over the relationships that are being projected. It takes two mandatory arguments: `graphName`, and `sourceNode`. The third parameter is `targetNode` and is usually provided. The parameter is optional and can be `null` to project an unconnected node. The next and fourth optional `dataConfig` parameter can be used to project node properties and labels as well as relationship properties and type. The last and fifth optional `configuration` parameter can be used for general configuration of the projection such as `readConcurrency`. -[.graph-project-cypher-aggregation-syntax] +[.graph-project-cypher-projection-syntax] -- [source, cypher, role=noplay] ---- @@ -69,12 +70,12 @@ RETURN gds.graph.project( | graphName | no | The name under which the graph is stored in the catalog. | sourceNode | no | The source node of the relationship. Must not be null. | targetNode | yes | The target node of the relationship. The targetNode can be null (for example due to an `OPTIONAL MATCH`), in which case the source node is projected as an unconnected node. -| <> | yes | Properties and labels configuration for the source and target nodes as well as properties and type configuration for the relationship. -| <> | yes | Additional parameters to configure the projection. +| <> | yes | Properties and labels configuration for the source and target nodes as well as properties and type configuration for the relationship. +| <> | yes | Additional parameters to configure the projection. |=== -[[graph-project-cypher-aggregation-syntax-dataConfig]] -.Nodes configuration +[[graph-project-cypher-projection-syntax-dataConfig]] +.Data configuration [opts="header",cols="1,1,1,4"] |=== | Name | Type | Default | Description @@ -86,7 +87,7 @@ RETURN gds.graph.project( | relationshipType | String | '*' | The type of the relationship. |=== -[[graph-project-cypher-aggregation-syntax-configuration]] +[[graph-project-cypher-projection-syntax-configuration]] .Configuration [opts="header",cols="1,1,1,4"] |=== @@ -112,7 +113,7 @@ RETURN gds.graph.project( NOTE: To get information about a stored graph, such as its schema, one can use xref:graph-list.adoc[gds.graph.list]. -[[graph-project-cypher-aggregation-examples]] +[[graph-project-cypher-projection-examples]] == Examples In order to demonstrate the GDS Cypher Aggregation we are going to create a small social network graph in Neo4j. @@ -139,7 +140,7 @@ CREATE ---- -[[graph-project-cypher-aggregation-example-single-label-type]] +[[graph-project-cypher-projection-example-single-label-type]] === Simple graph A simple graph is a graph with only one node label and relationship type, i.e., a monopartite graph. @@ -157,7 +158,7 @@ RETURN ---- .Results -[opts="header", cols="1,1,1m"] +[opts="header", cols="0,1,1m"] |=== | graph | nodes | rels | "persons" | 3 | 2 @@ -181,7 +182,7 @@ RETURN ---- .Results -[opts="header", cols="1,1,1m"] +[opts="header",cols="1,1,1m"] |=== | graph | nodes | rels | "persons" | 5 | 2 @@ -189,6 +190,7 @@ RETURN -- +[[graph-project-cypher-projection-arbitrary-source-and-target-id-values]] === Arbitrary source and target ID values So far, the examples showed how to project a graph based on existing nodes. @@ -197,7 +199,7 @@ It is also possible to pass INTEGER values directly. [role=query-example] -- .Project arbitrary id values: -[source, cypher, role=noplay] +[source,cypher,role=noplay] ---- UNWIND [ [42, 84], [13, 37], [19, 84] ] AS sourceAndTarget WITH sourceAndTarget[0] AS source, sourceAndTarget[1] AS target @@ -221,6 +223,7 @@ As such, `.write` procedures cannot be executed on this graph. -- +[[graph-project-cypher-projection-multi-graph]] === Multi-graph A multi-graph is a graph with multiple node labels and relationship types. @@ -313,6 +316,7 @@ RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels |=== -- +[[graph-project-cypher-projection-undirected-relationships]] ==== Undirected relationships Relationships can be projected as undirected by specifying the `undirectedRelationshipTypes` parameter. @@ -465,9 +469,10 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-aggregation.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. +We can see, that the `numberOfPages` are loaded. The default property value is `Double.Nan` and can be changed as in the previous example xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties] by using the Cypher function https://neo4j.com/docs/cypher-manual/current/functions/scalar/#functions-coalesce[_coalesce()_]. +[[graph-project-cypher-projection-parallel-relationships]] === Parallel relationships The Property Graph Model in Neo4j supports parallel relationships, i.e., multiple relationships between two nodes. @@ -579,6 +584,7 @@ ORDER BY numberOfPages DESC, person We can see, that the two `READ` relationships between Florentin and the Hobbit sum up to `46` numberOfPages. +[[graph-project-cypher-projection-filtered]] === Projecting filtered Neo4j graphs Cypher-projections allow us to specify the graph to project in a more fine-grained way. @@ -630,5 +636,5 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -If we compare the results to the ones from xref:management-ops/projections/graph-project-cypher-aggregation.adoc#cypher-aggregation-relationship-properties[Relationship properties], we can see that using `IS NOT NULL` is filtering out the relationship from Veselin to the book Frankenstein. +If we compare the results to the ones from xref:management-ops/projections/graph-project-cypher-projection.adoc#cypher-aggregation-relationship-properties[Relationship properties], we can see that using `IS NOT NULL` is filtering out the relationship from Veselin to the book Frankenstein. This functionality is only expressible with xref:management-ops/projections/graph-project.adoc[native projections] by projecting a xref:management-ops/projections/graph-project-subgraph.adoc[subgraph]. diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc index 0aabac01506..15867c18f60 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project.adoc @@ -3,8 +3,12 @@ :description: This section details projecting GDS graphs using `native` projections. -A native projection is the fastest and most scalable way to project a graph from a Neo4j database into the xref:management-ops/graph-catalog-ops.adoc[GDS Graph Catalog]. -Native projections are recommended for any use case, and for both the development and the production phase (see xref:common-usage/index.adoc[Common usage]). +Native projections are the easiest way to create a xref:management-ops/index.adoc[GDS graph] from a Neo4j database. +A native projection is entirely described by configuration parameters. + +_Node projections_ and _relationship projections_ describe the way nodes and relationships are loaded (_projected_) from the database into the in-memory graph. +While node projections are based on node labels, relationship projections are based on relationship types. +Both can include properties. == Considerations @@ -493,7 +497,7 @@ YIELD To specify the orientation, we need to write the `relationshipProjection` with the extended Map-syntax. Projecting the `KNOWS` relationships `UNDIRECTED`, loads each relationship in both directions. -Thus, the `undirectedKnows` graph contains four relationships, twice as many as the `persons` graph in xref:management-ops/projections/graph-project-cypher.adoc#graph-project-example-single-label-type[Simple graph]. +Thus, the `undirectedKnows` graph contains four relationships, twice as many as the `persons` graph in xref:management-ops/projections/graph-project.adoc#graph-project-example-single-label-type[Simple graph]. [[node-properties-example]] @@ -627,7 +631,7 @@ ORDER BY person ASC, numberOfPages DESC |=== -- -We can see, that the `numberOfPages` property is loaded. The default property value is `Double.NaN` and could be changed using the Map-Syntax the same as for node properties in xref:management-ops/projections/graph-project-cypher-aggregation.adoc#node-properties-example[Node properties]. +We can see, that the `numberOfPages` property is loaded. The default property value is `Double.NaN` and could be changed using the Map-Syntax the same as for node properties in xref:management-ops/projections/graph-project.adoc#node-properties-example[Node properties]. === Parallel relationships @@ -778,7 +782,7 @@ We can see, that the two READ relationships between Florentin and the Hobbit sum As mentioned in the xref:management-ops/projections/graph-project.adoc#graph-project-native-syntax[syntax section], the `validateRelationships` flag controls whether an error will be raised when attempting to project a relationship where either the source or target node is not present in the xref:management-ops/projections/graph-project.adoc#node-projection-syntax[node projection]. Note that even if the flag is set to `false` such a relationship will still not be projected but the loading process will not be aborted. -We can simulate such a case with the xref:management-ops/projections/graph-project-cypher.adoc#graph-project-examples[graph present in the Neo4j database]: +We can simulate such a case with the xref:management-ops/projections/graph-project.adoc#graph-project-examples[graph present in the Neo4j database]: -- .Project `READ` and `KNOWS` relationships but only `Person` nodes, with `validateRelationships` set to true: diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index bfd53b8d287..157de93cb23 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.4.0" +| "2.4.7" |=== -- diff --git a/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc new file mode 100644 index 00000000000..cf8df31d8e6 --- /dev/null +++ b/doc/modules/ROOT/pages/migration-alpha-cp-to-cpv2/index.adoc @@ -0,0 +1,185 @@ +[appendix] +[[appendix-d]] += Migration from Alpha Cypher Aggregation to new Cypher projection +:description: If you have been using `gds.alpha.graph.project` Cypher aggregation, you can find the info you will need to migrate to using the new Cypher projection. + + +== Who should read this guide + +This guide is intended for users who have been using the Alpha Cypher Aggregation https://neo4j.com/docs/graph-data-science/2.3/management-ops/projections/graph-project-cypher-aggregation/[`gds.alpha.graph.project`]. +Cypher projections are now done using the `gds.graph.project` aggregation function. +We assume that most of the mentioned operations and concepts can be understood with little explanation. +Thus we are intentionally brief in the examples and comparisons. +Please see xref:management-ops/projections/graph-project-cypher-projection.adoc[the documentation for the Cypher projection] for more details. + +== API Changes + +The new Cypher projection is a replacement for the Alpha Cypher Aggregation. +Like the Alpha Cypher Aggregation, the new Cypher projection is an aggregation function that is called as part of a Cypher query. + +The following changes have been made to the API: + +* The new Cypher projection is called using `gds.graph.project` instead of `gds.alpha.graph.project`. +* The new Cypher projection defines a single map parameter for defining projection related information such as labels or properties. +** There is still a separate map parameter for defining the graph configuration. +* The `properties` key of the relationship configuration map has been renamed to `relationshipProperties`. +* Additional validation to reduce mis-use of the API: +** Validate that each `sourceNode*` entry has a corresponding `targetNode*` entry and vice-versa. +** Identify end help with migration to this new API if any of the points above have not been followed. + + +.Structural changes between the Alpha aggregation/new projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +| +[source, cypher, role=noplay] +---- +$query +RETURN gds.alpha.graph.project( + $graphName, + sourceNode, + targetNode, + $nodeConfig, + $relationshipConfig, + $configuration +) +---- +| +[source, cypher, role=noplay] +---- +$relationshipQuery +RETURN gds.graph.project( + $graphName, + sourceNode, + targetNode, + $dataConfig, + $configuration +) +---- +|=== + +== Examples + +The following examples not include the full Cypher queries before the aggreation function is called, nor do they include and YIELD of return fields. +There are no changes related to those two aspects. + +.Side-by-side comparison +[opts=header,cols="1a,1a"] +|=== +| Alpha | New +2+| : Projection without any configuration +| +[source, cypher, role=noplay] +---- +... +RETURN gds.alpha.graph.project('g', source, target) +---- +| +[source, cypher, role=noplay] +---- +... +RETURN gds.graph.project('g', source, target) +---- +2+| : Multi-graph projection +| +[source, cypher, role=noplay] +---- +... +RETURN gds.alpha.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: labels(target), + }, { + relationshipType: type(r) + } +) +---- +| +[source, cypher, role=noplay] +---- +... +RETURN gds.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: labels(target), + relationshipType: type(rel) + } +) +---- +2+| : Graph projection with properties +| +[source, cypher, role=noplay] +---- +... +RETURN gds.alpha.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: labels(target), + sourceNodeProperties: source { .age }, + targetNodeProperties: target { .age }, + }, { + relationshipType: type(rel), + properties: rel { .numberOfPages } + } +) +---- +| +[source, cypher, role=noplay] +---- +... +RETURN gds.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: labels(target), + sourceNodeProperties: source { .age }, + targetNodeProperties: target { .age }, + relationshipType: type(rel), + relationshipProperties: rel { .numberOfPages } + } +) +---- +2+| : Graph projection with one-sided properties +| +[source, cypher, role=noplay] +---- +... +RETURN gds.alpha.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + sourceNodeProperties: source { .age }, + } +) +---- +| +[source, cypher, role=noplay] +---- +... +RETURN gds.graph.project( + 'g', + source, + target, + { + sourceNodeLabels: labels(source), + targetNodeLabels: NULL, + sourceNodeProperties: source { .age }, + targetNodeProperties: NULL, + } +) +---- +|=== diff --git a/doc/modules/ROOT/pages/appendix-b/index.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/index.adoc similarity index 60% rename from doc/modules/ROOT/pages/appendix-b/index.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/index.adoc index f67aa656758..07b4ae14156 100644 --- a/doc/modules/ROOT/pages/appendix-b/index.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/index.adoc @@ -2,6 +2,7 @@ [[appendix-b]] = Migration from Graph Data Science library Version 1.x :description: If you have previously used Graph Data Science library version 1.x, you can find the information you will need to migrate to using version 2.x in this section. +:page-aliases: appendix-b/index.adoc == Who should read this guide @@ -17,10 +18,10 @@ In this section we will focus on side-by-side examples of operations using the s This section is divided into the following sub-sections: -* xref:appendix-b/migration-algos-common.adoc[Common Changes] -* xref:appendix-b/migration-graph-projection.adoc[Graph Projection] -* xref:appendix-b/migration-graph-listing.adoc[Graph Listing] -* xref:appendix-b/migration-graph-drop.adoc[Graph Drop] -* xref:appendix-b/migration-memory-estimation.adoc[Memory Estimation] -* xref:appendix-b/migration-algorithms.adoc[Algorithms] -* xref:appendix-b/migration-ml.adoc[Machine Learning] +* xref:migration-gds-1-to-gds-2/migration-algos-common.adoc[Common Changes] +* xref:migration-gds-1-to-gds-2/migration-graph-projection.adoc[Graph Projection] +* xref:migration-gds-1-to-gds-2/migration-graph-listing.adoc[Graph Listing] +* xref:migration-gds-1-to-gds-2/migration-graph-drop.adoc[Graph Drop] +* xref:migration-gds-1-to-gds-2/migration-memory-estimation.adoc[Memory Estimation] +* xref:migration-gds-1-to-gds-2/migration-algorithms.adoc[Algorithms] +* xref:migration-gds-1-to-gds-2/migration-ml.adoc[Machine Learning] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-algorithms.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algorithms.adoc similarity index 92% rename from doc/modules/ROOT/pages/appendix-b/migration-algorithms.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algorithms.adoc index 9b8f6e21141..5a71604b200 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-algorithms.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algorithms.adoc @@ -1,6 +1,7 @@ [[migration-algorithms]] = Algorithms :description: This section covers migration for all algorithms in the Neo4j Graph Data Science library. +:page-aliases: appendix-b/migration-algorithms.adoc include::partial$/migration/migration-algorithms-betweenness-centrality.adoc[leveloffset=+1] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-algos-common.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algos-common.adoc similarity index 91% rename from doc/modules/ROOT/pages/appendix-b/migration-algos-common.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algos-common.adoc index 2b1bcf2ba19..3ff37a6d5d8 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-algos-common.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-algos-common.adoc @@ -1,5 +1,6 @@ [[migration-algos-common]] = Common changes +:page-aliases: appendix-b/migration-algos-common.adoc This section describes changes between version 1.x and 2.x that are common to all procedures. diff --git a/doc/modules/ROOT/pages/appendix-b/migration-graph-drop.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-drop.adoc similarity index 91% rename from doc/modules/ROOT/pages/appendix-b/migration-graph-drop.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-drop.adoc index fd3d085560e..e9a3425a8d8 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-graph-drop.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-drop.adoc @@ -1,5 +1,6 @@ [[migration-graph-drop]] = Graph drop +:page-aliases: appendix-b/migration-graph-drop.adoc .Changes in the YIELD fields [opts=header, cols="1,1"] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-graph-listing.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-listing.adoc similarity index 91% rename from doc/modules/ROOT/pages/appendix-b/migration-graph-listing.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-listing.adoc index af96427f8b3..db5e7a6bd9e 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-graph-listing.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-listing.adoc @@ -1,5 +1,6 @@ [[migration-graph-listing]] = Graph listing +:page-aliases: appendix-b/migration-graph-listing.adoc .Changes in the YIELD fields [opts=header, cols="1,1"] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-graph-projection.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-projection.adoc similarity index 96% rename from doc/modules/ROOT/pages/appendix-b/migration-graph-projection.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-projection.adoc index e185458e27e..876b02bc4d3 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-graph-projection.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-graph-projection.adoc @@ -1,5 +1,6 @@ [[migration-graph-projection]] = Graph projection +:page-aliases: appendix-b/migration-graph-projection.adoc .Changes in the YIELD fields [opts=header, cols="1,1"] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-memory-estimation.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-memory-estimation.adoc similarity index 95% rename from doc/modules/ROOT/pages/appendix-b/migration-memory-estimation.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-memory-estimation.adoc index 5e7f01ee2cd..1068b6bc92e 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-memory-estimation.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-memory-estimation.adoc @@ -1,5 +1,6 @@ [[migration-memory-estimation]] = Memory estimation +:page-aliases: appendix-b/migration-memory-estimation.adoc .Estimating memory for algorithms without loading the graph: [opts=header,cols="1a,1a"] diff --git a/doc/modules/ROOT/pages/appendix-b/migration-ml.adoc b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-ml.adoc similarity index 88% rename from doc/modules/ROOT/pages/appendix-b/migration-ml.adoc rename to doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-ml.adoc index 3550996e428..4b03f4681d4 100644 --- a/doc/modules/ROOT/pages/appendix-b/migration-ml.adoc +++ b/doc/modules/ROOT/pages/migration-gds-1-to-gds-2/migration-ml.adoc @@ -1,6 +1,7 @@ [[migration-ml]] = Machine Learning :description: This section covers migration for Machine Learning algorithms in the Neo4j Graph Data Science library. +:page-aliases: appendix-b/migration-ml.adoc include::partial$/migration/migration-algorithms-node-classification.adoc[leveloffset=+1] diff --git a/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc new file mode 100644 index 00000000000..d35f3acf382 --- /dev/null +++ b/doc/modules/ROOT/pages/migration-lcp-to-cpv2/index.adoc @@ -0,0 +1,469 @@ +[appendix] +[[appendix-c]] += Migration from Legacy to new Cypher projection +:description: If you have been using `gds.graph.project.cypher` projections, you can find the information you will need to migrate to using the new Cypher projection. + + +== Who should read this guide + +This guide is intended for users who have been using the Legacy Cypher projection xref:management-ops/projections/graph-project-cypher-legacy.adoc[`gds.graph.project.cypher`]. +Cypher projections are now done using the `gds.graph.project` aggregation function. +We assume that most of the mentioned operations and concepts can be understood with little explanation. +Thus we are intentionally brief in the examples and comparisons. +Please see xref:management-ops/projections/graph-project-cypher-projection.adoc[the documentation for the Cypher projection] for more details. + +== Structural Changes + +The Legacy Cypher projection is a standalone procedure call where Cypher queries are passed as string arguments and executed by GDS. +The new Cypher projection is an aggregation function that is called as part of a Cypher query. +GDS is no longer responsible or in control of the execution of the Cypher queries. +Migrating to the new Cypher projection requires changes of how the Cypher query is written as a whole. + +There are no longer separate queries for nodes and relationships. +Instead, write one query that produces the source- and target node pairs and use `gds.graph.project` to aggregate into the graph catalog. +Since the relationship query from the Legacy Cypher projection already required you to return the source- and target node pairs, it is a good starting point for the new query. +Roughly speaking, the query has to be rewritten as follows: + +.Structural changes between the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + $graphName, + $nodeQuery, + $relationshipQuery, + $configuration +) +---- +| +[source, cypher, role=noplay] +---- +$relationshipQuery +RETURN gds.graph.project( + $graphName, + sourceNode, + targetNode, + $dataConfig, + $configuration +) +---- +|=== + +The query no longer needs to adhere to a certain structure and you can use any Cypher query that produces the source- and target node pairs. + +== Semantic Changes + +The Legacy Cypher projections has separate queries for nodes and relationships. +The nodes query is executed first and defines all the nodes in the graph. +The relationships query is executed second and the previously imported nodes act as a filter for the relationships. +Only relationships between the previously imported nodes are imported into the graph. +Any node that was imported as part of the node query, but does not appear in any of the relationships, results in a disconnected node in the graph. +By default, all nodes are disconnected unless they also appear in a relationship. + +The new Cypher projection does not have separate queries for nodes and relationships. +The node query is no longer needed and nodes are implicitly created from the source- and target node pairs. +Disconnected nodes have to be explicitly created in the query by providing `NULL` in place of the target node. +By default, all nodes are connected unless they are explicitly disconnected. + +Since the new Cypher projection is no longer in charge of executing the Cypher queries, the graph configuration can no longer return the node- and relationship queries. + +== Examples + +The following examples are based on the examples listed in the documentation for xref:management-ops/projections/graph-project-cypher-legacy.adoc[the Legacy Cypher projection] and xref:management-ops/projections/graph-project-cypher-projection.adoc[the new Cypher projection]. + +=== Simple graph + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Simple graph projection with potentially disconnected nodes +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'persons', + 'MATCH (n:Person) RETURN id(n) AS id', + 'MATCH (n:Person)-[r:KNOWS]->(m:Person) RETURN id(n) AS source, id(m) AS target') +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n:Person) +OPTIONAL MATCH (n)-[r:KNOWS]->(m:Person) +WITH gds.graph.project('persons', n, m) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +2+| : Simple graph projection without disconnected nodes +| +Not applicable, Legacy Cypher projection cannot guarantee connected nodes. +| +[source, cypher, role=noplay] +---- +MATCH (n:Person)-[r:KNOWS]->(m:Person) +WITH gds.graph.project('persons', n, m) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +|=== + +The direct translation requires the use of an `OPTIONAL MATCH` clause to create disconnected nodes in order to create an identical graph. +This may not have been what you wanted originally, but was required since the Legacy Cypher projection could not guarantee connected nodes. +By using what is equivalent to the `$relationshipQuery`, we now also get only connected nodes in the new Cypher projection. + +Another difference is that we pass the nodes directly to the new Cypher projection. +The Legacy Cypher projection required us to pass the node ids. +By passing the nodes directly, the Cypher projection knows that the source for the projection is a Neo4j database and it enables the use of `.write` procedures. +It is also possible to pass node ids instead of nodes `... gds.graph.project('persons', id(n), id(m))`, but this is only recommended if the source for the projection is not a neo4j database. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-arbitrary-source-and-target-id-values[Arbitrary source and target id values] for more details. + +=== Multi-graph + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Multi-graph projection +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'personsAndBooks', + 'MATCH (n) WHERE n:Person OR n:Book RETURN id(n) AS id, labels(n) AS labels', + 'MATCH (n)-[r:KNOWS\|READ]->(m) RETURN id(n) AS source, id(m) AS target, type(r) AS type') +YIELD + graphName AS graph, nodeQuery, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n) +WHERE n:Person OR n:Book +OPTIONAL MATCH (n)-[r:KNOWS\|READ]->(m) +WHERE m:Person OR m:Book +WITH gds.graph.project( + 'personsAndBooks', + n, + m, + { + sourceNodeLabels: labels(n), + targetNodeLabels: labels(m), + relationshipType: type(r) + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +|=== + +Similar to the previous example, we have to use an `OPTIONAL MATCH` clause to create disconnected nodes in order to create an identical graph. +The query can also look different depending on the actual graph schema and whether disconnected nodes are desired. + +Node labels and relationship types are passed as an additional configuration map to the new Cypher projection. +Node labels need to be passed as `sourceNodeLabels` and `targetNodeLabels` and relationship types need to be passed as `relationshipType`. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-multi-graph[Multi-graph] for more details. + +=== Node properties + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with node properties +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'graphWithProperties', + 'MATCH (n:Person) + RETURN + id(n) AS id, + labels(n) AS labels, + n.age AS age', + 'MATCH (n)-[r:KNOWS]->(m) RETURN id(n) AS source, id(m) AS target, type(r) AS type' +) +YIELD + graphName, nodeCount AS nodes, relationshipCount AS rels +RETURN graphName, nodes, rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n:Person) +OPTIONAL MATCH (n)-[r:KNOWS]->(m:Person) +WITH gds.graph.project( + 'graphWithProperties', + n, + m, + { + sourceNodeLabels: labels(n), + targetNodeLabels: labels(m), + sourceNodeProperties: n { .age }, + targetNodeProperties: m { .age }, + relationshipType: type(r) + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +2+| : Graph projection with optional node properties +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'graphWithProperties', + 'MATCH (n) + WHERE n:Book OR n:Person + RETURN + id(n) AS id, + labels(n) AS labels, + coalesce(n.age, 18) AS age', + coalesce(n.price, 5.0) AS price, + n.ratings AS ratings', + 'MATCH (n)-[r:KNOWS\|READ]->(m) RETURN id(n) AS source, id(m) AS target, type(r) AS type' +) +YIELD + graphName, nodeCount AS nodes, relationshipCount AS rels +RETURN graphName, nodes, rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n) +WHERE n:Person OR n:Book +OPTIONAL MATCH (n)-[r:KNOWS\|READ]->(m) +WHERE m:Person OR m:Book +WITH gds.graph.project( + 'graphWithProperties', + n, + m, + { + sourceNodeLabels: labels(n), + targetNodeLabels: labels(m), + sourceNodeProperties: n { age: coalesce(n.age, 18), price: coalesce(n.price, 5.0), .ratings }, + targetNodeProperties: n { age: coalesce(n.age, 18), price: coalesce(n.price, 5.0), .ratings }, + relationshipType: type(r) + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS node, g.relationshipCount AS rels +---- +|=== + +Similar to the previous example, we pass the labels and properties in an additional map. +We can use map projections as well as any other Cypher expression to create the properties. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#node-properties-example[Node properties] for more details. + + +=== Relationship properties + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with relationship properties +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'readWithProperties', + 'MATCH (n) RETURN id(n) AS id', + 'MATCH (n)-[r:READ]->(m) + RETURN id(n) AS source, id(m) AS target, r.numberOfPages AS numberOfPages' +) +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n)-[r:READ]->(m) +WITH gds.graph.project( + 'readWithProperties', + n, + m, + { relationshipProperties: r { .numberOfPages } } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels + +---- +|=== + +Similar to the previous example, we pass properties in an additional map, here using the `relationshipProperties` key. +We can use map projections as well as any other Cypher expression to create the properties. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#cypher-aggregation-relationship-properties[Relationship properties] for more details. + + +=== Parallel Relationship + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with parallel relationships +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'readCount', + 'MATCH (n) RETURN id(n) AS id', + 'MATCH (n)-[r:READ]->(m) + RETURN id(n) AS source, id(m) AS target, type(r) AS type, count(r) AS numberOfReads' +) +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n)-[r:READ]->(m) +WITH n, m, count(r) AS numberOfReads +WITH gds.graph.project( + 'readCount', + n, + m, + { + relationshipProperties: { numberOfReads: numberOfReads } + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels +---- +2+| : Graph projection with parallel relationship and relationship properties +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'readSums', + 'MATCH (n) RETURN id(n) AS id', + 'MATCH (n)-[r:READ]->(m) + RETURN id(n) AS source, id(m) AS target, sum(r.numberOfPages) AS numberOfPages' +) +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n)-[r:READ]->(m) +WITH n, m, sum(r.numberOfPages) AS numberOfPages +WITH gds.graph.project( + 'readSums', + n, + m, + { + relationshipProperties: { numberOfPages: numberOfPages } + } +) AS g +RETURN + g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels +---- +|=== + +Similar to Legacy Cypher projections, there is no mechanism to let GDS aggregate parallel relationships. +Aggregations over parallel relationships are done in the query by any means that are appropriate for the graph schema and data. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-parallel-relationships[Parallel relationship] for more details. + + +=== Projecting filtered graphs + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with filtered graphs +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher( + 'existingNumberOfPages', + 'MATCH (n) RETURN id(n) AS id', + 'MATCH (n)-[r:READ]->(m) + WHERE r.numberOfPages IS NOT NULL + RETURN id(n) AS source, id(m) AS target, r.numberOfPages AS numberOfPages' +) +YIELD + graphName AS graph, nodeCount AS nodes, relationshipCount AS rels +---- +| +[source, cypher, role=noplay] +---- +MATCH (n) OPTIONAL MATCH (n)-[r:READ]->(m) +WHERE r.numberOfPages IS NOT NULL +WITH gds.graph.project('existingNumberOfPages', n, m, { relationshipProperties: r { .numberOfPages } }) AS g +RETURN + g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels + +---- +|=== + +Similar to Legacy Cypher projections, we can apply any Cypher method of filtering the data before passing it on to the Cypher projection. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-filtered[Projecting filtered Neo4j graphs] for more details. + + +=== Projecting undirected graphs + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Graph projection with undirected graphs +| +Not applicable, Legacy Cypher projection cannot project undirected graphs. +| +[source, cypher, role=noplay] +---- +MATCH (n)-[r:KNOWS\|READ]->(m) +WHERE n:Book OR n:Person +WITH gds.graph.project( + 'graphWithUndirectedRelationships', + source, + target, + {}, + {undirectedRelationshipTypes: ['*']} +) AS g +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels +---- +|=== + +The new Cypher projection can project undirected graphs. +See xref:management-ops/projections/graph-project-cypher-projection.adoc#graph-project-cypher-projection-undirected-relationships[Undirected relationships] for more details. + + +=== Memory estimation + +.Side-by-side comparison of the two Cypher projections: +[opts=header,cols="1a,1a"] +|=== +| Legacy | New +2+| : Memory estimation of projected graphs +| +[source, cypher, role=noplay] +---- +CALL gds.graph.project.cypher.estimate( + 'MATCH (n:Person) RETURN id(n) AS id', + 'MATCH (n:Person)-[r:KNOWS]->(m:Person) RETURN id(n) AS source, id(m) AS target' +) YIELD requiredMemory, bytesMin, bytesMax +---- +| +[source, cypher, role=noplay] +---- +MATCH (n:Person)-[r:KNOWS]-(m) +WITH count(n) AS nodeCount, count(r) AS relationshipCount +CALL gds.graph.project.estimate('*', '*', { + nodeCount: nodeCount, + relationshipCount: relationshipCount, +}) +YIELD requiredMemory, bytesMin, bytesMax +---- +|=== + +Since the new Cypher projection is no longer a procedure, there is also no `.estimate` method. +Instead, we can use xref:common-usage/memory-estimation.adoc#estimate-procedure-fictive-graph[the `gds.graph.project.estimate` procedure] to estimate the memory requirements of the graph projection. diff --git a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc index 5753bcc6277..fac60faafc2 100644 --- a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc @@ -21,6 +21,7 @@ | xref:graph-catalog-node-ops.adoc#utility-functions-catalog[Accessing a node property in a named graph] | `_gds.util.nodeProperty_` | xref:alpha-algorithms/one-hot-encoding.adoc[One Hot Encoding] | `_gds.alpha.ml.oneHotEncoding_` | xref:common-usage/debug-sysinfo.adoc[Status of the system] | `gds.debug.sysInfo` +| xref:installation/configure-apache-arrow-server.adoc[Monitoring] | `gds.debug.arrow` | xref:management-ops/create-cypher-db.adoc[Create an impermanent database backed by a projected graph] | `gds.alpha.create.cypherdb` | xref:management-ops/create-cypher-db.adoc#drop-cypher-db[Drop an impermanent database backed by a projected graph] | `gds.alpha.drop.cypherdb` | xref:common-usage/monitoring-system.adoc[Get an overview of the system's workload and available resources] | `gds.alpha.systemMonitor` diff --git a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc index a7833b9992f..20d3e93c5e8 100644 --- a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc @@ -11,6 +11,106 @@ [opts=header,cols="1, 2"] |=== | Algorithm name | Operation +.8+<.^| xref:algorithms/delta-single-source.adoc[All Shortest Paths Delta-Stepping] +| `gds.allShortestPaths.delta.stream` +| `gds.allShortestPaths.delta.stream.estimate` +| `gds.allShortestPaths.delta.write` +| `gds.allShortestPaths.delta.write.estimate` +| `gds.allShortestPaths.delta.mutate` +| `gds.allShortestPaths.delta.mutate.estimate` +| `gds.allShortestPaths.delta.stats` +| `gds.allShortestPaths.delta.stats.estimate` +.6+<.^| xref:algorithms/dijkstra-single-source.adoc[All Shortest Paths Dijkstra] +| `gds.allShortestPaths.dijkstra.stream` +| `gds.allShortestPaths.dijkstra.stream.estimate` +| `gds.allShortestPaths.dijkstra.write` +| `gds.allShortestPaths.dijkstra.write.estimate` +| `gds.allShortestPaths.dijkstra.mutate` +| `gds.allShortestPaths.dijkstra.mutate.estimate` +.8+<.^|xref:algorithms/article-rank.adoc[ArticleRank] +| `gds.articleRank.mutate` +| `gds.articleRank.mutate.estimate` +| `gds.articleRank.write` +| `gds.articleRank.write.estimate` +| `gds.articleRank.stream` +| `gds.articleRank.stream.estimate` +| `gds.articleRank.stats` +| `gds.articleRank.stats.estimate` +.8+<.^| xref:algorithms/bellman-ford-single-source.adoc[Bellman-Ford] +| `gds.bellmanFord.mutate` +| `gds.bellmanFord.mutate.estimate` +| `gds.bellmanFord.stats` +| `gds.bellmanFord.stats.estimate` +| `gds.bellmanFord.stream` +| `gds.bellmanFord.stream.estimate` +| `gds.bellmanFord.write` +| `gds.bellmanFord.write.estimate` +.8+<.^| xref:algorithms/betweenness-centrality.adoc#algorithms-betweenness-centrality-syntax[Betweenness Centrality] +| `gds.betweenness.stream` +| `gds.betweenness.stream.estimate` +| `gds.betweenness.stats` +| `gds.betweenness.stats.estimate` +| `gds.betweenness.mutate` +| `gds.betweenness.mutate.estimate` +| `gds.betweenness.write` +| `gds.betweenness.write.estimate` +.6+<.^|xref:algorithms/bfs.adoc[Breadth First Search] +| `gds.bfs.mutate` +| `gds.bfs.mutate.estimate` +| `gds.bfs.stream` +| `gds.bfs.stream.estimate` +| `gds.bfs.stats` +| `gds.bfs.stats.estimate` +.8+<.^| xref:algorithms/degree-centrality.adoc[Degree Centrality] +| `gds.degree.mutate` +| `gds.degree.mutate.estimate` +| `gds.degree.stats` +| `gds.degree.stats.estimate` +| `gds.degree.stream` +| `gds.degree.stream.estimate` +| `gds.degree.write` +| `gds.degree.write.estimate` +.4+<.^|xref:algorithms/dfs.adoc[Depth First Search] +| `gds.dfs.mutate` +| `gds.dfs.mutate.estimate` +| `gds.dfs.stream` +| `gds.dfs.stream.estimate` +.8+<.^|xref:algorithms/eigenvector-centrality.adoc[Eigenvector] +| `gds.eigenvector.mutate` +| `gds.eigenvector.mutate.estimate` +| `gds.eigenvector.write` +| `gds.eigenvector.write.estimate` +| `gds.eigenvector.stream` +| `gds.eigenvector.stream.estimate` +| `gds.eigenvector.stats` +| `gds.eigenvector.stats.estimate` +.8+<.^| xref:machine-learning/node-embeddings/fastrp.adoc[Fast Random Projection] +| `gds.fastRP.mutate` +| `gds.fastRP.mutate.estimate` +| `gds.fastRP.stats` +| `gds.fastRP.stats.estimate` +| `gds.fastRP.stream` +| `gds.fastRP.stream.estimate` +| `gds.fastRP.write` +| `gds.fastRP.write.estimate` +.8+<.^| xref:algorithms/k-core.adoc[K-Core Decomposition] +| `gds.kcore.stats` +| `gds.kcore.stats.estimate` +| `gds.kcore.stream` +| `gds.kcore.stream.estimate` +| `gds.kcore.mutate` +| `gds.kcore.mutate.estimate` +| `gds.kcore.write` +| `gds.kcore.write.estimate` +.8+<.^|xref:algorithms/knn.adoc[K-Nearest Neighbors] +| `gds.knn.mutate` +| `gds.knn.mutate.estimate` +| `gds.knn.stats` +| `gds.knn.stats.estimate` +| `gds.knn.stream` +| `gds.knn.stream.estimate` +| `gds.knn.write` +| `gds.knn.write.estimate` .8+<.^|xref:algorithms/label-propagation.adoc#algorithms-label-propagation-syntax[Label Propagation] | `gds.labelPropagation.mutate` | `gds.labelPropagation.mutate.estimate` @@ -20,6 +120,15 @@ | `gds.labelPropagation.stream.estimate` | `gds.labelPropagation.stats` | `gds.labelPropagation.stats.estimate` +.8+<.^| xref:algorithms/local-clustering-coefficient.adoc#algorithms-local-clustering-coefficient-syntax[Local Clustering Coefficient] +| `gds.localClusteringCoefficient.stream` +| `gds.localClusteringCoefficient.stream.estimate` +| `gds.localClusteringCoefficient.stats` +| `gds.localClusteringCoefficient.stats.estimate` +| `gds.localClusteringCoefficient.write` +| `gds.localClusteringCoefficient.write.estimate` +| `gds.localClusteringCoefficient.mutate` +| `gds.localClusteringCoefficient.mutate.estimate` .8+<.^|xref:algorithms/louvain.adoc#algorithms-louvain-syntax[Louvain] | `gds.louvain.mutate` | `gds.louvain.mutate.estimate` @@ -47,87 +156,27 @@ | `gds.pageRank.stream.estimate` | `gds.pageRank.stats` | `gds.pageRank.stats.estimate` -.8+<.^|xref:algorithms/wcc.adoc#algorithms-wcc-syntax[Weakly Connected Components] -| `gds.wcc.mutate` -| `gds.wcc.mutate.estimate` -| `gds.wcc.write` -| `gds.wcc.write.estimate` -| `gds.wcc.stream` -| `gds.wcc.stream.estimate` -| `gds.wcc.stats` -| `gds.wcc.stats.estimate` -.8+<.^| xref:algorithms/triangle-count.adoc#algorithms-triangle-count-syntax[Triangle Count] -| `gds.triangleCount.stream` -| `gds.triangleCount.stream.estimate` -| `gds.triangleCount.stats` -| `gds.triangleCount.stats.estimate` -| `gds.triangleCount.write` -| `gds.triangleCount.write.estimate` -| `gds.triangleCount.mutate` -| `gds.triangleCount.mutate.estimate` -.8+<.^| xref:algorithms/local-clustering-coefficient.adoc#algorithms-local-clustering-coefficient-syntax[Local Clustering Coefficient] -| `gds.localClusteringCoefficient.stream` -| `gds.localClusteringCoefficient.stream.estimate` -| `gds.localClusteringCoefficient.stats` -| `gds.localClusteringCoefficient.stats.estimate` -| `gds.localClusteringCoefficient.write` -| `gds.localClusteringCoefficient.write.estimate` -| `gds.localClusteringCoefficient.mutate` -| `gds.localClusteringCoefficient.mutate.estimate` -.8+<.^| xref:algorithms/betweenness-centrality.adoc#algorithms-betweenness-centrality-syntax[Betweenness Centrality] -| `gds.betweenness.stream` -| `gds.betweenness.stream.estimate` -| `gds.betweenness.stats` -| `gds.betweenness.stats.estimate` -| `gds.betweenness.mutate` -| `gds.betweenness.mutate.estimate` -| `gds.betweenness.write` -| `gds.betweenness.write.estimate` -.8+<.^| xref:machine-learning/node-embeddings/fastrp.adoc[Fast Random Projection] -| `gds.fastRP.mutate` -| `gds.fastRP.mutate.estimate` -| `gds.fastRP.stats` -| `gds.fastRP.stats.estimate` -| `gds.fastRP.stream` -| `gds.fastRP.stream.estimate` -| `gds.fastRP.write` -| `gds.fastRP.write.estimate` -.8+<.^| xref:algorithms/degree-centrality.adoc[Degree Centrality] -| `gds.degree.mutate` -| `gds.degree.mutate.estimate` -| `gds.degree.stats` -| `gds.degree.stats.estimate` -| `gds.degree.stream` -| `gds.degree.stream.estimate` -| `gds.degree.write` -| `gds.degree.write.estimate` -.8+<.^|xref:algorithms/article-rank.adoc[ArticleRank] -| `gds.articleRank.mutate` -| `gds.articleRank.mutate.estimate` -| `gds.articleRank.write` -| `gds.articleRank.write.estimate` -| `gds.articleRank.stream` -| `gds.articleRank.stream.estimate` -| `gds.articleRank.stats` -| `gds.articleRank.stats.estimate` -.8+<.^|xref:algorithms/eigenvector-centrality.adoc[Eigenvector] -| `gds.eigenvector.mutate` -| `gds.eigenvector.mutate.estimate` -| `gds.eigenvector.write` -| `gds.eigenvector.write.estimate` -| `gds.eigenvector.stream` -| `gds.eigenvector.stream.estimate` -| `gds.eigenvector.stats` -| `gds.eigenvector.stats.estimate` -.8+<.^| xref:algorithms/delta-single-source.adoc[All Shortest Paths Delta-Stepping] -| `gds.allShortestPaths.delta.stream` -| `gds.allShortestPaths.delta.stream.estimate` -| `gds.allShortestPaths.delta.write` -| `gds.allShortestPaths.delta.write.estimate` -| `gds.allShortestPaths.delta.mutate` -| `gds.allShortestPaths.delta.mutate.estimate` -| `gds.allShortestPaths.delta.stats` -| `gds.allShortestPaths.delta.stats.estimate` +.4+<.^|xref:algorithms/random-walk.adoc[Random Walk] +| `gds.randomWalk.stats` +| `gds.randomWalk.stats.estimate` +| `gds.randomWalk.stream` +| `gds.randomWalk.stream.estimate` +.8+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] +| `gds.scaleProperties.mutate` +| `gds.scaleProperties.mutate.estimate` +| `gds.scaleProperties.stream` +| `gds.scaleProperties.stream.estimate` +| `gds.scaleProperties.stats` +| `gds.scaleProperties.stats.estimate` +| `gds.scaleProperties.write` +| `gds.scaleProperties.write.estimate` +.6+<.^| xref:algorithms/astar.adoc[Shortest Path AStar] +| `gds.shortestPath.astar.stream` +| `gds.shortestPath.astar.stream.estimate` +| `gds.shortestPath.astar.write` +| `gds.shortestPath.astar.write.estimate` +| `gds.shortestPath.astar.mutate` +| `gds.shortestPath.astar.mutate.estimate` .6+<.^| xref:algorithms/dijkstra-source-target.adoc[Shortest Path Dijkstra] | `gds.shortestPath.dijkstra.stream` | `gds.shortestPath.dijkstra.stream.estimate` @@ -135,13 +184,6 @@ | `gds.shortestPath.dijkstra.write.estimate` | `gds.shortestPath.dijkstra.mutate` | `gds.shortestPath.dijkstra.mutate.estimate` -.6+<.^| xref:algorithms/dijkstra-single-source.adoc[All Shortest Paths Dijkstra] -| `gds.allShortestPaths.dijkstra.stream` -| `gds.allShortestPaths.dijkstra.stream.estimate` -| `gds.allShortestPaths.dijkstra.write` -| `gds.allShortestPaths.dijkstra.write.estimate` -| `gds.allShortestPaths.dijkstra.mutate` -| `gds.allShortestPaths.dijkstra.mutate.estimate` .6+<.^| xref:algorithms/yens.adoc[Shortest Paths Yens] | `gds.shortestPath.yens.stream` | `gds.shortestPath.yens.stream.estimate` @@ -149,46 +191,31 @@ | `gds.shortestPath.yens.write.estimate` | `gds.shortestPath.yens.mutate` | `gds.shortestPath.yens.mutate.estimate` -.6+<.^| xref:algorithms/astar.adoc[Shortest Path AStar] -| `gds.shortestPath.astar.stream` -| `gds.shortestPath.astar.stream.estimate` -| `gds.shortestPath.astar.write` -| `gds.shortestPath.astar.write.estimate` -| `gds.shortestPath.astar.mutate` -| `gds.shortestPath.astar.mutate.estimate` .6+<.^|xref:algorithms/similarity-functions.adoc[Similarity functions] -| `gds.similarity.cosine` -| `gds.similarity.euclidean` -| `gds.similarity.euclideanDistance` -| `gds.similarity.jaccard` -| `gds.similarity.overlap` -| `gds.similarity.pearson` -.8+<.^|xref:algorithms/knn.adoc[K-Nearest Neighbors] -| `gds.knn.mutate` -| `gds.knn.mutate.estimate` -| `gds.knn.stats` -| `gds.knn.stats.estimate` -| `gds.knn.stream` -| `gds.knn.stream.estimate` -| `gds.knn.write` -| `gds.knn.write.estimate` -.6+<.^|xref:algorithms/bfs.adoc[BFS] -| `gds.bfs.mutate` -| `gds.bfs.mutate.estimate` -| `gds.bfs.stream` -| `gds.bfs.stream.estimate` -| `gds.bfs.stats` -| `gds.bfs.stats.estimate` -.4+<.^|xref:algorithms/dfs.adoc[Depth First Search] -| `gds.dfs.mutate` -| `gds.dfs.mutate.estimate` -| `gds.dfs.stream` -| `gds.dfs.stream.estimate` -.4+<.^|xref:algorithms/random-walk.adoc[Random Walk] -| `gds.randomWalk.stats` -| `gds.randomWalk.stats.estimate` -| `gds.randomWalk.stream` -| `gds.randomWalk.stream.estimate` +| `_gds.similarity.cosine_` +| `_gds.similarity.euclidean_` +| `_gds.similarity.euclideanDistance_` +| `_gds.similarity.jaccard_` +| `_gds.similarity.overlap_` +| `_gds.similarity.pearson_` +.8+<.^| xref:algorithms/triangle-count.adoc#algorithms-triangle-count-syntax[Triangle Count] +| `gds.triangleCount.stream` +| `gds.triangleCount.stream.estimate` +| `gds.triangleCount.stats` +| `gds.triangleCount.stats.estimate` +| `gds.triangleCount.write` +| `gds.triangleCount.write.estimate` +| `gds.triangleCount.mutate` +| `gds.triangleCount.mutate.estimate` +.8+<.^|xref:algorithms/wcc.adoc#algorithms-wcc-syntax[Weakly Connected Components] +| `gds.wcc.mutate` +| `gds.wcc.mutate.estimate` +| `gds.wcc.write` +| `gds.wcc.write.estimate` +| `gds.wcc.stream` +| `gds.wcc.stream.estimate` +| `gds.wcc.stats` +| `gds.wcc.stats.estimate` |=== [[beta-tier]] @@ -216,6 +243,20 @@ | `gds.beta.graphSage.write.estimate` | `gds.beta.graphSage.train` | `gds.beta.graphSage.train.estimate` +.4+<.^|xref:machine-learning/node-embeddings/hashgnn.adoc[HashGNN] +| `gds.beta.hashgnn.mutate` +| `gds.beta.hashgnn.mutate.estimate` +| `gds.beta.hashgnn.stream` +| `gds.beta.hashgnn.stream.estimate` +.8+<.^| xref:algorithms/celf.adoc[Influence Maximization - CELF] +| `gds.beta.influenceMaximization.celf.mutate` +| `gds.beta.influenceMaximization.celf.mutate.estimate` +| `gds.beta.influenceMaximization.celf.stats` +| `gds.beta.influenceMaximization.celf.stats.estimate` +| `gds.beta.influenceMaximization.celf.stream` +| `gds.beta.influenceMaximization.celf.stream.estimate` +| `gds.beta.influenceMaximization.celf.write` +| `gds.beta.influenceMaximization.celf.write.estimate` .8+<.^|xref:algorithms/k1coloring.adoc[K1Coloring] | `gds.beta.k1coloring.mutate` | `gds.beta.k1coloring.mutate.estimate` @@ -225,6 +266,29 @@ | `gds.beta.k1coloring.stream.estimate` | `gds.beta.k1coloring.write` | `gds.beta.k1coloring.write.estimate` +.8+<.^| xref:algorithms/kmeans.adoc[Kmeans] +| `gds.beta.kmeans.mutate` +| `gds.beta.kmeans.mutate.estimate` +| `gds.beta.kmeans.stats` +| `gds.beta.kmeans.stats.estimate` +| `gds.beta.kmeans.stream` +| `gds.beta.kmeans.stream.estimate` +| `gds.beta.kmeans.write` +| `gds.beta.kmeans.write.estimate` +.8+<.^| xref:algorithms/leiden.adoc[Leiden] +| `gds.beta.leiden.mutate` +| `gds.beta.leiden.mutate.estimate` +| `gds.beta.leiden.stats` +| `gds.beta.leiden.stats.estimate` +| `gds.beta.leiden.stream` +| `gds.beta.leiden.stream.estimate` +| `gds.beta.leiden.write` +| `gds.beta.leiden.write.estimate` +.4+<.^| xref:algorithms/directed-steiner-tree.adoc[Minimum Directed Steiner Tree] +| `gds.beta.steinerTree.mutate` +| `gds.beta.steinerTree.stats` +| `gds.beta.steinerTree.stream` +| `gds.beta.steinerTree.write` .6+<.^| xref:algorithms/modularity-optimization.adoc[Modularity Optimization] | `gds.beta.modularityOptimization.mutate` | `gds.beta.modularityOptimization.mutate.estimate` @@ -239,25 +303,15 @@ | `gds.beta.node2vec.stream.estimate` | `gds.beta.node2vec.write` | `gds.beta.node2vec.write.estimate` -.8+<.^| xref:algorithms/celf.adoc[Influence Maximization - CELF] -| `gds.beta.influenceMaximization.celf.mutate` -| `gds.beta.influenceMaximization.celf.mutate.estimate` -| `gds.beta.influenceMaximization.celf.stats` -| `gds.beta.influenceMaximization.celf.stats.estimate` -| `gds.beta.influenceMaximization.celf.stream` -| `gds.beta.influenceMaximization.celf.stream.estimate` -| `gds.beta.influenceMaximization.celf.write` -| `gds.beta.influenceMaximization.celf.write.estimate` -.8+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] -| `gds.scaleProperties.mutate` -| `gds.scaleProperties.mutate.estimate` -| `gds.scaleProperties.stream` -| `gds.scaleProperties.stream.estimate` -| `gds.scaleProperties.stats` -| `gds.scaleProperties.stats.estimate` -| `gds.scaleProperties.write` -| `gds.scaleProperties.write.estimate` - +.8+<.^|xref:algorithms/minimum-weight-spanning-tree.adoc[Spanning Tree] +| `gds.beta.spanningTree.mutate` +| `gds.beta.spanningTree.mutate.estimate` +| `gds.beta.spanningTree.stats` +| `gds.beta.spanningTree.stats.estimate` +| `gds.beta.spanningTree.stream` +| `gds.beta.spanningTree.stream.estimate` +| `gds.beta.spanningTree.write` +| `gds.beta.spanningTree.write.estimate` |=== [[alpha-tier]] @@ -269,6 +323,7 @@ [opts=header,cols="1, 2"] |=== |Algorithm name | Operation +| xref:alpha-algorithms/adamic-adar.adoc[Adamic Adar] | `_gds.alpha.linkprediction.adamicAdar_` .1+<.^|xref:alpha-algorithms/all-pairs-shortest-path.adoc[All Shortest Paths] | `gds.alpha.allShortestPaths.stream` .4+<.^|xref:algorithms/alpha/approx-max-k-cut.adoc[Approximate Maximum k-cut] @@ -276,6 +331,23 @@ | `gds.alpha.maxkcut.mutate.estimate` | `gds.alpha.maxkcut.stream` | `gds.alpha.maxkcut.stream.estimate` +| xref:alpha-algorithms/common-neighbors.adoc[Common Neighbors] | `_gds.alpha.linkprediction.commonNeighbors_` +.1+<.^| xref:algorithms/alpha/conductance.adoc[Conductance] +| `gds.alpha.conductance.stream` +.4+<.^| xref:algorithms/alpha/filtered-knn.adoc[Filtered KNN] +| `gds.alpha.knn.filtered.mutate` +| `gds.alpha.knn.filtered.stats` +| `gds.alpha.knn.filtered.stream` +| `gds.alpha.knn.filtered.write` +.8+<.^| xref:algorithms/alpha/filtered-node-similarity.adoc[Filtered NodeSimilarity] +| `gds.alpha.nodeSimilarity.filtered.mutate` +| `gds.alpha.nodeSimilarity.filtered.mutate.estimate` +| `gds.alpha.nodeSimilarity.filtered.stats` +| `gds.alpha.nodeSimilarity.filtered.stats.estimate` +| `gds.alpha.nodeSimilarity.filtered.stream` +| `gds.alpha.nodeSimilarity.filtered.stream.estimate` +| `gds.alpha.nodeSimilarity.filtered.write` +| `gds.alpha.nodeSimilarity.filtered.write.estimate` .2+<.^|xref:algorithms/harmonic-centrality.adoc[Harmonic Centrality] | `gds.alpha.closeness.harmonic.stream` | `gds.alpha.closeness.harmonic.write` @@ -288,9 +360,11 @@ | `gds.alpha.hits.stream.estimate` | `gds.alpha.hits.write` | `gds.alpha.hits.write.estimate` -.2+<.^|xref:algorithms/strongly-connected-components.adoc[Strongly Connected Components] -| `gds.alpha.scc.stream` -| `gds.alpha.scc.write` +.1+<.^|xref:alpha-algorithms/k-minimum-weight-spanning-tree.adoc[ k-Spanning Tree] +| `gds.alpha.kSpanningTree.write` +| xref:alpha-algorithms/preferential-attachment.adoc[Preferential Attachment] | `_gds.alpha.linkprediction.preferentialAttachment_` +| xref:alpha-algorithms/resource-allocation.adoc[Resource Allocation] | `_gds.alpha.linkprediction.resourceAllocation_` +| xref:alpha-algorithms/same-community.adoc[Same Community] | `_gds.alpha.linkprediction.sameCommunity_` .8+<.^|xref:algorithms/sllpa.adoc[Speaker-Listener Label Propagation] | `gds.alpha.sllpa.mutate` | `gds.alpha.sllpa.mutate.estimate` @@ -300,93 +374,15 @@ | `gds.alpha.sllpa.stream.estimate` | `gds.alpha.sllpa.write` | `gds.alpha.sllpa.write.estimate` -.8+<.^|xref:algorithms/minimum-weight-spanning-tree.adoc[Spanning Tree] -| `gds.beta.spanningTree.mutate` -| `gds.beta.spanningTree.mutate.estimate` -| `gds.beta.spanningTree.stats` -| `gds.beta.spanningTree.stats.estimate` -| `gds.beta.spanningTree.stream` -| `gds.beta.spanningTree.stream.estimate` -| `gds.beta.spanningTree.write` -| `gds.beta.spanningTree.write.estimate` -.1+<.^|xref:alpha-algorithms/k-minimum-weight-spanning-tree.adoc[ k-Spanning Tree] -| `gds.alpha.kSpanningTree.write` -| xref:alpha-algorithms/adamic-adar.adoc[Adamic Adar] | `_gds.alpha.linkprediction.adamicAdar_` -| xref:alpha-algorithms/common-neighbors.adoc[Common Neighbors] | `_gds.alpha.linkprediction.commonNeighbors_` -| xref:alpha-algorithms/preferential-attachment.adoc[Preferential Attachment] | `_gds.alpha.linkprediction.preferentialAttachment_` -| xref:alpha-algorithms/preferential-attachment.adoc[Preferential Attachment] | `_gds.alpha.linkprediction.resourceAllocation_` -| xref:alpha-algorithms/same-community.adoc[Same Community] | `_gds.alpha.linkprediction.sameCommunity_` -| xref:alpha-algorithms/total-neighbors.adoc[Total Neighbors] | `_gds.alpha.linkprediction.totalNeighbors_` .1+<.^| xref:alpha-algorithms/split-relationships.adoc[Split Relationships] | `gds.alpha.ml.splitRelationships.mutate` +.2+<.^|xref:algorithms/strongly-connected-components.adoc[Strongly Connected Components] +| `gds.alpha.scc.stream` +| `gds.alpha.scc.write` .1+<.^| xref:algorithms/triangle-count.adoc#algorithms-triangle-count-examples-triangles-listing[Triangle Listing] | `gds.alpha.triangles` -.1+<.^| xref:algorithms/alpha/conductance.adoc[Conductance] -| `gds.alpha.conductance.stream` -.4+<.^|xref:machine-learning/node-embeddings/hashgnn.adoc[HashGNN] -| `gds.beta.hashgnn.mutate` -| `gds.beta.hashgnn.mutate.estimate` -| `gds.beta.hashgnn.stream` -| `gds.beta.hashgnn.stream.estimate` -.8+<.^| xref:algorithms/kmeans.adoc[Kmeans] -| `gds.beta.kmeans.mutate` -| `gds.beta.kmeans.mutate.estimate` -| `gds.beta.kmeans.stats` -| `gds.beta.kmeans.stats.estimate` -| `gds.beta.kmeans.stream` -| `gds.beta.kmeans.stream.estimate` -| `gds.beta.kmeans.write` -| `gds.beta.kmeans.write.estimate` -.4+<.^| xref:algorithms/alpha/filtered-knn.adoc[Filtered KNN] -| `gds.alpha.knn.filtered.mutate` -| `gds.alpha.knn.filtered.stats` -| `gds.alpha.knn.filtered.stream` -| `gds.alpha.knn.filtered.write` -.8+<.^| xref:algorithms/leiden.adoc[Leiden] -| `gds.beta.leiden.mutate` -| `gds.beta.leiden.mutate.estimate` -| `gds.beta.leiden.stats` -| `gds.beta.leiden.stats.estimate` -| `gds.beta.leiden.stream` -| `gds.beta.leiden.stream.estimate` -| `gds.beta.leiden.write` -| `gds.beta.leiden.write.estimate` -.8+<.^| xref:algorithms/alpha/filtered-node-similarity.adoc[Filtered NodeSimilarity] -| `gds.alpha.nodeSimilarity.filtered.mutate` -| `gds.alpha.nodeSimilarity.filtered.mutate.estimate` -| `gds.alpha.nodeSimilarity.filtered.stats` -| `gds.alpha.nodeSimilarity.filtered.stats.estimate` -| `gds.alpha.nodeSimilarity.filtered.stream` -| `gds.alpha.nodeSimilarity.filtered.stream.estimate` -| `gds.alpha.nodeSimilarity.filtered.write` -| `gds.alpha.nodeSimilarity.filtered.write.estimate` +| xref:alpha-algorithms/total-neighbors.adoc[Total Neighbors] | `_gds.alpha.linkprediction.totalNeighbors_` .2+<.^| xref:algorithms/alpha/modularity.adoc[Modularity Metric] | `gds.alpha.modularity.stats` | `gds.alpha.modularity.stream` -.4+<.^| xref:algorithms/directed-steiner-tree.adoc[Minimum Directed Steiner Tree] -| `gds.beta.steinerTree.mutate` -| `gds.beta.steinerTree.stats` -| `gds.beta.steinerTree.stream` -| `gds.beta.steinerTree.write` -.8+<.^| xref:algorithms/bellman-ford-single-source.adoc[Bellman-Ford] -| `gds.bellmanFord.mutate` -| `gds.bellmanFord.mutate.estimate` -| `gds.bellmanFord.stats` -| `gds.bellmanFord.stats.estimate` -| `gds.bellmanFord.stream` -| `gds.bellmanFord.stream.estimate` -| `gds.bellmanFord.write` -| `gds.bellmanFord.write.estimate` -.2+<.^|xref:algorithms/scale-properties.adoc[Scale Properties] -| `gds.alpha.scaleProperties.mutate` (deprecated) -| `gds.alpha.scaleProperties.stream` (deprecated) -.8+<.^| xref:algorithms/k-core.adoc[K-Core Decomposition] -| `gds.kcore.stats` -| `gds.kcore.stats.estimate` -| `gds.kcore.stream` -| `gds.kcore.stream.estimate` -| `gds.kcore.mutate` -| `gds.kcore.mutate.estimate` -| `gds.kcore.write` -| `gds.kcore.write.estimate` |=== diff --git a/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc index bfda78154d1..5eda06b775e 100644 --- a/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc @@ -11,8 +11,8 @@ .5+<.^|xref:management-ops/projections/graph-project.adoc[Project Graph] | `gds.graph.project` | `gds.graph.project.estimate` -| `gds.graph.project.cypher` -| `gds.graph.project.cypher.estimate` +| `gds.graph.project.cypher` (deprecated) +| `gds.graph.project.cypher.estimate` (deprecated) | `gds.graph.project` (aggregation function) .2+<.^|xref:graph-exists.adoc[Check if a graph exists] | `gds.graph.exists` diff --git a/doc/modules/ROOT/pages/production-deployment/composite.adoc b/doc/modules/ROOT/pages/production-deployment/composite.adoc index f6587c5424e..887670fa6bd 100644 --- a/doc/modules/ROOT/pages/production-deployment/composite.adoc +++ b/doc/modules/ROOT/pages/production-deployment/composite.adoc @@ -105,7 +105,7 @@ For every row returned by the first subquery, the operational database is then q In this mode of using GDS in a composite environment, the GDS operations are executed on the Fabric proxy server. The graph projections are then using the data stored on the shards to construct the in-memory graph. -NOTE: Currently only xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] is supported for projecting in-memory graphs on a Composite database. +NOTE: Currently only xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projection] is supported for projecting in-memory graphs on a Composite database. Graph algorithms can then be executed on the composite database, similar to a single machine setup. This scenario is useful, if a graph that logically represents a single graph is distributed to different Composite shards. diff --git a/doc/modules/ROOT/pages/production-deployment/feature-toggles.adoc b/doc/modules/ROOT/pages/production-deployment/feature-toggles.adoc index f2b1fac5e6d..96dd7fc79fc 100644 --- a/doc/modules/ROOT/pages/production-deployment/feature-toggles.adoc +++ b/doc/modules/ROOT/pages/production-deployment/feature-toggles.adoc @@ -20,20 +20,6 @@ To switch to the more memory intensive implementation used in GDS Community Edit CALL gds.features.useBitIdMap(false) ---- -[.enterprise-edition] -[[sharded-id-map-feature-toggle]] -== ShardedIdMap Feature Toggle - -The xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[BitIdMap] is optimized for a low memory footprint when used together with a Neo4j database. -However, its memory footprint is not optimal if the range of possible original node ids exceeds the node count significantly. -In this situation the `ShardedIdMap` can be used to significantly reduce the required memory of the graph projection. -To enable the sharded id map the following procedure call can be used: - -[source, cypher, role=noplay] ----- -CALL gds.features.useShardedIdMap(true) ----- - [[packed-adjacency-list-feature-toggle]] == Packed Adjacency List Toggle diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 27db2bca5bf..9df8805c176 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -29,7 +29,7 @@ GDS has compute-intensive OLAP workloads that may disrupt the cluster operations [NOTE] ====== -Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/[official Neo4j documentation] for details on how to setup Neo4j cluster. +Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/setup/analytics-cluster/[official Neo4j documentation] for details on how to set up a Neo4j analytics cluster. Note that the link points to the latest Neo4j version documentation and the configuration settings may differ from earlier versions. ====== diff --git a/doc/modules/ROOT/partials/algorithms/node-similarity/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/node-similarity/specific-configuration.adoc index 27a68af42cd..89849147971 100644 --- a/doc/modules/ROOT/partials/algorithms/node-similarity/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/node-similarity/specific-configuration.adoc @@ -24,5 +24,5 @@ This value cannot be negative, a value of 0 means no global limit. If unspecified, the algorithm runs unweighted. | similarityMetric | String | JACCARD | yes | The metric used to compute similarity. -Can be either `JACCARD` or `OVERLAP`. +Can be either `JACCARD`, `OVERLAP` or `COSINE`. |=== diff --git a/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc b/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc index a96646d7165..30a72a0ef8e 100644 --- a/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc +++ b/doc/modules/ROOT/partials/algorithms/shared/examples-named-native-note.adoc @@ -1,5 +1,5 @@ [NOTE] ==== In the examples below we will use named graphs and native projections as the norm. -However, xref:management-ops/projections/graph-project-cypher.adoc[Cypher projections] can also be used. +However, xref:management-ops/projections/graph-project-cypher-projection.adoc[Cypher projections] can also be used. ==== diff --git a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc index 858700857ba..a0432ed6ca1 100644 --- a/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc +++ b/doc/modules/ROOT/partials/management-ops/graph-catalog/graph-export-csv.adoc @@ -48,9 +48,10 @@ YIELD |=== | Name | Type | Default | Optional | Description | exportName | String | none | No | The name of the directory where the graph is exported to. The absolute path of the exported CSV files depends on the configuration parameter `gds.export.location` in the `neo4j.conf`. -| writeConcurrency | Boolean | 4 | yes | The number of concurrent threads used for writing the database. +| writeConcurrency | Integer | 4 | yes | The number of concurrent threads used for writing the database. | defaultRelationshipType | String | +__ALL__+ | yes | Relationship type used for `*` relationship projections. | additionalNodeProperties | String, List or Map | {} | yes | Allows for exporting additional node properties from the original graph backing the projected graph. +| useLabelMapping | Boolean | false | yes | Flag to decide whether to use node label mapping when exporting the graph |=== @@ -107,7 +108,7 @@ YIELD | Name | Type | Default | Optional | Description | exportName | String | none | no | Name of the folder the exported CSV files are saved at. | samplingFactor | Double | 0.001 | yes | The fraction of nodes and relationships to sample for the estimation. -| writeConcurrency | Boolean | 4 | yes | The number of concurrent threads used for writing the database. +| writeConcurrency | Integer | 4 | yes | The number of concurrent threads used for writing the database. | defaultRelationshipType | String | +__ALL__+ | yes | Relationship type used for `*` relationship projections. |=== @@ -132,6 +133,11 @@ YIELD The format of the exported CSV files is based on the format that is supported by the Neo4j Admin https://neo4j.com/docs/operations-manual/current/tools/neo4j-admin/neo4j-admin-import/[import] command. +GDS does not add a column for node labels and relationship types in the data. +In order to import them using Neo4j Admin, the labels and types should be set using the `--nodes` and `--relationship` parameters. + +More details here: using the same https://neo4j.com/docs/operations-manual/current/tutorial/neo4j-admin-import/#_using_the_same_label_for_every_node[label for every node], +https://neo4j.com/docs/operations-manual/current/tutorial/neo4j-admin-import/#_using_the_same_relationship_type_for_every_relationship[type for every relationship] === Nodes @@ -160,6 +166,32 @@ nodes_A_B_1.csv nodes_A_B_2.csv ---- +==== Nodes label mapping [[node_label_mapping]] + +When the configuration parameter `useLabelMapping` is set to true, the names of the labels will be mapped to integers during the export. +This mapping will be written to a new file named `label-mappings.csv`. +This parameter is required when label names contain underscores or special characters that are forbidden in file names by the OS. + +Using the example above, if label mapping is enabled, the content of `label-mappings.csv` may be: +---- +index,label +0,A +1,B +---- + +In this case, these files will be created for the nodes: + +---- +nodes_0_header.csv +nodes_0_0.csv +nodes_1_header.csv +nodes_1_0.csv +nodes_1_2.csv +nodes_0_1_header.csv +nodes_0_1_0.csv +nodes_0_1_1.csv +nodes_0_1_2.csv +---- === Relationships diff --git a/doc/package.json b/doc/package.json index 0923bc03028..5df0a3afb39 100644 --- a/doc/package.json +++ b/doc/package.json @@ -9,9 +9,7 @@ "start:publish": "nodemon -e adoc --exec \"npm run build:publish && npm run serve\"", "serve": "node server.js", "build": "antora preview.yml --stacktrace --log-format=pretty", - "build:publish": "antora publish.yml --stacktrace --log-format=pretty", - "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json", - "publish-verify": "antora --stacktrace --fetch publish.yml --log-format=json --log-file ./build/log/log.json" + "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json" }, "keywords": [ "antora", diff --git a/doc/preview.yml b/doc/preview.yml index 1901f52d69c..7a25f90bdd6 100644 --- a/doc/preview.yml +++ b/doc/preview.yml @@ -16,7 +16,7 @@ content: - '!**/README.adoc' ui: bundle: - url: https://s3-eu-west-1.amazonaws.com/static-content.neo4j.com/build/ui-bundle-latest.zip + url: https://static-content.neo4j.com/build/ui-bundle-latest.zip snapshot: true output_dir: /assets diff --git a/doc/publish.yml b/doc/publish.yml deleted file mode 100644 index d31048ec23a..00000000000 --- a/doc/publish.yml +++ /dev/null @@ -1,60 +0,0 @@ -site: - title: Neo4j Graph Data Science - url: https://neo4j.com/docs - start_page: graph-data-science:ROOT:index.adoc - -content: - sources: - - url: https://github.com/neo-technology/graph-analytics.git - branches: ['2.3', 'master'] - start_path: public/doc - include: public/doc/ - exclude: - - '!**/_includes/*' - - '!**/readme.adoc' - - '!**/README.adoc' -ui: - bundle: - url: https://s3-eu-west-1.amazonaws.com/static-content.neo4j.com/build/ui-bundle-latest.zip - snapshot: true - output_dir: /assets - -urls: - html_extension_style: indexify - -antora: - extensions: - - require: "@neo4j-antora/antora-modify-sitemaps" - sitemap_version: '2.3' - sitemap_loc_version: current - move_sitemaps_to_components: 'true' - -asciidoc: - extensions: - - "@neo4j-documentation/remote-include" - - "@neo4j-documentation/macros" - - "@neo4j-antora/antora-page-roles" - - "@neo4j-antora/antora-table-footnotes" - attributes: - page-theme: docs - page-type: Docs - page-search-type: Docs - page-search-site: Reference Docs - page-canonical-root: /docs - page-pagination: true - page-no-canonical: true - page-origin-private: true - page-hide-toc: false - page-mixpanel: 4bfb2414ab973c741b6f067bf06d5575 - # page-cdn: /static/assets - includePDF: false - sectnums: true - sectnumlevel: 3 - doctype: book - nonhtmloutput: "" - # sectnums: true, removed so they are off by default - # sectnumlevel: 3, - experimental: '' - copyright: 2023 - common-license-page-uri: https://neo4j.com/docs/license/ - operations-manual-base-uri: https://neo4j.com/docs/operations-manual/ diff --git a/etc/forbidden-apis b/etc/forbidden-apis index 87ef4e5d7c6..97d13d3492e 100644 --- a/etc/forbidden-apis +++ b/etc/forbidden-apis @@ -1,6 +1,6 @@ org.neo4j.internal.helpers.collection.MapUtil @ Use the GDS MapUtil instead. -java.lang.Thread.Thread(java.lang.Runnable) @ Use org.neo4j.gds.core.concurrency.Pools.newThread to make sure that the new thread is properly named -java.lang.Thread.Thread(java.lang.ThreadGroup, java.lang.Runnable) @ Use org.neo4j.gds.core.concurrency.Pools.newThread to make sure that the new thread is properly named +java.lang.Thread.Thread(java.lang.Runnable) @ Use org.neo4j.gds.core.concurrency.ExecutorServiceUtil.newThread to make sure that the new thread is properly named +java.lang.Thread.Thread(java.lang.ThreadGroup, java.lang.Runnable) @ Use org.neo4j.gds.core.concurrency.ExecutorServiceUtil.newThread to make sure that the new thread is properly named org.neo4j.internal.batchimport.staging.ExecutionMonitor.Adapter @ Implement CompatExecutionMonitor and call Neo4jProxy.executionMonitor org.neo4j.logging.internal.LogService#getUserLogProvider() @ Pass the logService through and use Neo4jProxy at the end. diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index c7dc7cd32f8..d0bf8c797fd 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,8 +7,8 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.4.0' - neo4jVersion = '5.8.0' + gdsVersion = '2.4.6' + neo4jVersion = '5.10.0' // Necessary to generate value classes for Pregel configs immutablesVersion = '2.8.1' @@ -28,6 +28,7 @@ dependencies { // We depend on the proc artifact of the GDS library compileOnly "org.neo4j.gds:proc:$gdsVersion" compileOnly "org.neo4j.gds:core:$gdsVersion" + compileOnly "org.neo4j.gds:core-write:$gdsVersion" compileOnly "org.neo4j.gds:algo-common:$gdsVersion" compileOnly "org.neo4j.gds:executor:$gdsVersion" compileOnly "org.neo4j.gds:memory-usage:$gdsVersion" @@ -40,6 +41,7 @@ dependencies { // We also need to depend on Neo4j itself for our tests. // Usually, Neo4j is available when we are running as a plugin. compileOnly "org.neo4j:neo4j:$neo4jVersion" + compileOnly "org.neo4j:neo4j-procedure-api:$neo4jVersion" // We need an annotation processor for our K1 configuration annotationProcessor "org.immutables:value:$immutablesVersion" diff --git a/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.jar b/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.jar index c1962a79e29..033e24c4cdf 100644 Binary files a/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.jar and b/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.jar differ diff --git a/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.properties b/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.properties index a21c6ebe28b..c747538fb38 100644 --- a/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.properties +++ b/examples/pregel-bootstrap/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-all.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/examples/pregel-bootstrap/gradlew b/examples/pregel-bootstrap/gradlew index aeb74cbb43e..fcb6fca147c 100755 --- a/examples/pregel-bootstrap/gradlew +++ b/examples/pregel-bootstrap/gradlew @@ -130,10 +130,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. diff --git a/examples/pregel-bootstrap/src/main/java/gds/example/ExamplePregelComputation.java b/examples/pregel-bootstrap/src/main/java/gds/example/ExamplePregelComputation.java index b06510b75b2..4778e4ec8f6 100644 --- a/examples/pregel-bootstrap/src/main/java/gds/example/ExamplePregelComputation.java +++ b/examples/pregel-bootstrap/src/main/java/gds/example/ExamplePregelComputation.java @@ -32,7 +32,7 @@ import org.neo4j.gds.beta.pregel.context.InitContext; import org.neo4j.gds.core.CypherMapWrapper; -@PregelProcedure(name = "pregel.example", modes = {GDSMode.STREAM}, description = "My first Pregel example") +@PregelProcedure(name = "pregel.example", modes = {GDSMode.STREAM, GDSMode.WRITE}, description = "My first Pregel example") public class ExamplePregelComputation implements PregelComputation { public static final String KEY = "key"; diff --git a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java index 26f51bb3952..7143e16fcbd 100644 --- a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java +++ b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationAlgoTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.TestSupport; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -64,7 +64,7 @@ void runExamplePregelComputation() { graph, config, new ExamplePregelComputation(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationWriteProcTest.java b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationWriteProcTest.java new file mode 100644 index 00000000000..600e7cfd8c4 --- /dev/null +++ b/examples/pregel-bootstrap/src/test/java/gds/example/ExamplePregelComputationWriteProcTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017-2020 "Neo4j," + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package gds.example; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.Neo4jGraph; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.LONG; + +class ExamplePregelComputationWriteProcTest extends BaseProcTest { + + @Neo4jGraph + private static final String MY_TEST_GRAPH = + "CREATE" + + " (alice)" + + ", (bob)" + + ", (eve)" + + ", (alice)-[:LIKES]->(bob)" + + ", (bob)-[:LIKES]->(alice)" + + ", (eve)-[:DISLIKES]->(alice)" + + ", (eve)-[:DISLIKES]->(bob)"; + + @BeforeEach + void setup() throws Exception { + registerProcedures(ExamplePregelComputationWriteProc.class, GraphProjectProc.class); + } + + @Test + void write() { + runQueryWithRowConsumer("MATCH (n) WHERE n.pregelkey IS NOT NULL RETURN count(n) AS count ", row -> { + assertThat(row.getNumber("count")) + .asInstanceOf(LONG) + .as("There should be no nodes having the `pregelkey` property") + .isEqualTo(0); + }); + + runQuery("CALL gds.graph.project('graph', '*', '*')"); + runQuery("CALL pregel.example.write('graph', { maxIterations: 10, writeProperty: 'pregel' })"); + + runQueryWithRowConsumer("MATCH (n) WHERE n.pregelkey IS NOT NULL RETURN count(n) AS count ", row -> { + assertThat(row.getNumber("count")) + .asInstanceOf(LONG) + .as("All nodes should have the `pregelkey` property") + .isEqualTo(3); + }); + } +} diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java index 5acceeec466..af7d71fbe7d 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/bfs/BFSPregelAlgoTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.TestSupport; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -111,7 +111,7 @@ void levelBfs() { graph, config, new BFSLevelPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -147,7 +147,7 @@ void parentBfs() { graph, config, new BFSParentPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -183,7 +183,7 @@ void parentBugTest() { parentGraph, config, new BFSParentPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java index c8f0c96109c..856755c1f9d 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/cc/ConnectedComponentsPregelAlgoTest.java @@ -23,7 +23,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.TestSupport; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -85,7 +85,7 @@ void wcc() { graph, config, new ConnectedComponentsPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java index e9ebb5a069d..4a6611a9f43 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/lp/LabelPropagationPregelAlgoTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -73,7 +73,7 @@ void runLP() { graph, config, new LabelPropagationPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -111,7 +111,7 @@ public double applyRelationshipWeight(double nodeValue, double relationshipWeigh graph, config, weightedLabelPropagation, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java index febe951e386..86b211d519e 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/PageRankPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -88,7 +88,7 @@ void runPR() { graph, config, new PageRankPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java index 42fa414694e..c0e44d4f749 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/pr/WeightedPageRankPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -130,7 +130,7 @@ static void assertResult(TestGraph graph, Map expected) { graph, config, new PageRankPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java index da597122bec..48566f62d83 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/sssp/SingleSourceShortestPathPregelAlgoTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; @@ -83,7 +83,7 @@ void runSSSP() { graph, config, new SingleSourceShortestPathPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java index fd3e4d30988..5582660c6e2 100644 --- a/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java +++ b/examples/pregel-example/src/test/java/org/neo4j/gds/beta/pregel/triangleCount/TriangleCountPregelTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.Pregel; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -332,7 +332,7 @@ HugeLongArray nodeWiseTriangles(Graph graph) { graph, config, new TriangleCountPregel(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/executor/src/main/java/org/neo4j/gds/executor/AlgoConfigParser.java b/executor/src/main/java/org/neo4j/gds/executor/AlgoConfigParser.java index aa7681fa50e..6cb6d25a78d 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/AlgoConfigParser.java +++ b/executor/src/main/java/org/neo4j/gds/executor/AlgoConfigParser.java @@ -26,11 +26,11 @@ import org.neo4j.gds.configuration.LimitsConfiguration; import org.neo4j.gds.core.CypherMapWrapper; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.Map; -import java.util.Set; public class AlgoConfigParser implements ProcConfigParser { private final NewConfigFunction newConfigFunction; @@ -53,28 +53,13 @@ public AlgoConfigParser( @Override public CONFIG processInput(Map configuration) { // apply defaults - var configurationWithDefaultsApplied = defaults.apply(configuration, username); - + var inputWithDefaults = applyDefaults(configuration, username); // parse configuration - CONFIG algorithmConfiguration = produceConfig(configurationWithDefaultsApplied); - - // handle limits - var allowedKeys = new HashSet<>(algorithmConfiguration.configKeys()); - var irrelevantInputtedKeys = getIrrelevantInputtedKeys(configuration, allowedKeys); - var configurationButWithIrrelevantInputtedKeysRemoved = getConfigurationForLimitValidation( - configuration, - irrelevantInputtedKeys - ); - validateLimits(configurationButWithIrrelevantInputtedKeysRemoved); - - // validate configuration - var addedKeys = findAddedKeys(configurationWithDefaultsApplied, configuration.keySet()); - var addedButIrrelevantKeys = findIrrelevantKeys(addedKeys, allowedKeys); - var configurationWithDefaultsAppliedButIrrelevantKeysRemoved = weedOutIrrelevantKeys( - configurationWithDefaultsApplied, - addedButIrrelevantKeys - ); - validateConfig(allowedKeys, configurationWithDefaultsAppliedButIrrelevantKeysRemoved); + CONFIG algorithmConfiguration = produceConfig(inputWithDefaults); + //validate the original config for extra-added parameters + validateOriginalConfig(configuration, algorithmConfiguration.configKeys()); + //ensure that limits are fine + validateLimits(algorithmConfiguration, username, inputWithDefaults); return algorithmConfiguration; } @@ -96,6 +81,25 @@ private HashMap getConfigurationForLimitValidation( return configurationButWithIrrelevantInputtedKeysRemoved; } + void validateLimits( + CONFIG algorithmConfiguration, + String username, + Map userInputWithDefaults + ) throws IllegalArgumentException { + + // handle limits + var allowedKeys = new HashSet<>(algorithmConfiguration.configKeys()); + var irrelevantInputtedKeys = getIrrelevantInputtedKeys(userInputWithDefaults, allowedKeys); + var configurationButWithIrrelevantInputtedKeysRemoved = getConfigurationForLimitValidation( + userInputWithDefaults, + irrelevantInputtedKeys + ); //remove any useless configuration parameters e.g., sourceNode for Wcc + + validateLimits(configurationButWithIrrelevantInputtedKeysRemoved); + + } + + private void validateLimits(HashMap configurationButWithIrrelevantInputtedKeysRemoved) { var violations = limits.validate(configurationButWithIrrelevantInputtedKeysRemoved, username); @@ -121,31 +125,19 @@ private CONFIG produceConfig(Map configurationWithDefaultsApplie return newConfigFunction.apply(username, cypherMapWrapper); } - @NotNull - private Set findAddedKeys(Map configurationWithDefaultsApplied, Set inputtedKeys) { - Set addedKeys = new HashSet<>(configurationWithDefaultsApplied.keySet()); - addedKeys.removeAll(inputtedKeys); - return addedKeys; - } - @NotNull - private Set findIrrelevantKeys(Set addedKeys, Set allowedKeys) { - Set irrelevantKeys = new HashSet<>(addedKeys); - irrelevantKeys.removeAll(allowedKeys); - return irrelevantKeys; - } + void validateOriginalConfig( + Map configuration, + Collection allowedConfigKeys + ) throws IllegalArgumentException { + Map newConfiguration = new HashMap<>(configuration); - @NotNull - private HashMap weedOutIrrelevantKeys( - Map configurationWithDefaultsApplied, - Set irrelevantKeys - ) { - var configurationWithDefaultsAppliedButIrrelevantKeysRemoved = new HashMap<>(configurationWithDefaultsApplied); - configurationWithDefaultsAppliedButIrrelevantKeysRemoved.keySet().removeAll(irrelevantKeys); - return configurationWithDefaultsAppliedButIrrelevantKeysRemoved; + CypherMapWrapper.create(newConfiguration) + .requireOnlyKeysFrom(allowedConfigKeys); //ensure user has not included any incorrect params } - private void validateConfig(Set allowedKeys, Map configuration) { - CypherMapWrapper.create(configuration).requireOnlyKeysFrom(allowedKeys); + Map applyDefaults(Map configuration, String username) { + // apply defaults + return defaults.apply(configuration, username); } } diff --git a/executor/src/main/java/org/neo4j/gds/executor/ComputationResult.java b/executor/src/main/java/org/neo4j/gds/executor/ComputationResult.java index 170acbf4df7..fdbd6e5624e 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/ComputationResult.java +++ b/executor/src/main/java/org/neo4j/gds/executor/ComputationResult.java @@ -38,6 +38,10 @@ public interface ComputationResult, ALGO_RESULT @Nullable A algorithm(); + /** + * Result is empty if no computation happened, which basically means the graph was empty. + * @return The result if computation happened. + */ Optional result(); Graph graph(); diff --git a/executor/src/main/java/org/neo4j/gds/executor/ComputationResultConsumer.java b/executor/src/main/java/org/neo4j/gds/executor/ComputationResultConsumer.java index aef3533876f..5916236c81c 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/ComputationResultConsumer.java +++ b/executor/src/main/java/org/neo4j/gds/executor/ComputationResultConsumer.java @@ -25,9 +25,4 @@ @FunctionalInterface public interface ComputationResultConsumer, ALGO_RESULT, CONFIG extends AlgoBaseConfig, RESULT> { RESULT consume(ComputationResult computationResult, ExecutionContext executionContext); - - static , ALGO_RESULT, CONFIG extends AlgoBaseConfig> - ComputationResultConsumer> identity() { - return (result, executionContext) -> result; - } } diff --git a/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java b/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java index 43c74760477..ee333204b6b 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java +++ b/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java @@ -25,10 +25,10 @@ import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.TerminationMonitor; +import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -120,7 +120,7 @@ public DatabaseId databaseId() { @Override public DependencyResolver dependencyResolver() { - return EmptyDependencyResolver.INSTANCE; + return Neo4jProxy.emptyDependencyResolver(); } @Override diff --git a/executor/src/test/java/org/neo4j/gds/executor/AlgoConfigParserTest.java b/executor/src/test/java/org/neo4j/gds/executor/AlgoConfigParserTest.java index 05064842969..4024ed5c707 100644 --- a/executor/src/test/java/org/neo4j/gds/executor/AlgoConfigParserTest.java +++ b/executor/src/test/java/org/neo4j/gds/executor/AlgoConfigParserTest.java @@ -29,6 +29,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatException; import static org.junit.jupiter.api.Assertions.fail; class AlgoConfigParserTest { @@ -196,4 +197,28 @@ void shouldReportAllLimitViolationsInOneGo() { " - Configuration parameter 'qux' with value '117' exceeds it's limit of '87'"); } } + + + @Test + void shouldComplainWhenAddedDefaultsDoNotAdhereToLimits() { + var configurationParser = new AlgoConfigParser<>( + "Miltos Tentoglou", + (NewConfigFunction) (username, config) -> new FooConfigImpl(config), + new DefaultsConfiguration( + Map.of("bar", new Default(16L), "sudo", new Default(false)), + Collections.emptyMap() + ), + new LimitsConfiguration( + Map.of("bar", LimitFactory.create(8L), "sudo", LimitFactory.create(true)), + Collections.emptyMap() + ) + ); + + assertThatException().isThrownBy(() -> configurationParser.processInput(Map.of())).withMessage( + "Configuration exceeded multiple limits:\n" + + " - Configuration parameter 'bar' with value '16' exceeds it's limit of '8'\n" + + " - Configuration parameter 'sudo' with value 'false' is in violation of it's set limit"); + + + } } diff --git a/executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java b/executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java index f72a836fab6..95936b218be 100644 --- a/executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java +++ b/executor/src/test/java/org/neo4j/gds/executor/MemoryEstimationExecutorTest.java @@ -25,12 +25,12 @@ import org.neo4j.gds.BaseTest; import org.neo4j.gds.GdsCypher; import org.neo4j.gds.NodeProjections; -import org.neo4j.gds.ProcedureCallContextReturnColumns; import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.NodeLookup; +import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.TerminationMonitor; import org.neo4j.gds.catalog.GraphProjectProc; import org.neo4j.gds.compat.GraphDatabaseApiProxy; @@ -47,7 +47,6 @@ import org.neo4j.gds.test.TestMutateConfig; import org.neo4j.gds.transaction.DatabaseTransactionContext; import org.neo4j.graphdb.Transaction; -import org.neo4j.internal.kernel.api.procs.ProcedureCallContext; import java.util.List; import java.util.Map; @@ -74,13 +73,7 @@ void setup() throws Exception { .builder() .databaseId(DatabaseId.of(db)) .dependencyResolver(GraphDatabaseApiProxy.dependencyResolver(db)) - .returnColumns(new ProcedureCallContextReturnColumns(new ProcedureCallContext( - 42, - new String[0], - false, - "neo4j", - false - ))) + .returnColumns(ProcedureReturnColumns.EMPTY) .log(Neo4jProxy.testLog()) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) diff --git a/executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java b/executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java index 3fb3a9af6e7..4e7c4bb04e9 100644 --- a/executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java +++ b/executor/src/test/java/org/neo4j/gds/executor/ProcedureExecutorTest.java @@ -25,7 +25,6 @@ import org.neo4j.gds.ProcedureCallContextReturnColumns; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.TerminationMonitor; @@ -131,7 +130,7 @@ private ExecutionContext executionContext(TaskStore taskStore) { .username("") .terminationMonitor(TerminationMonitor.EMPTY) .isGdsAdmin(true) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .modelCatalog(ModelCatalog.EMPTY) .closeableResourceRegistry(CloseableResourceRegistry.EMPTY) .algorithmMetaDataSetter(AlgorithmMetaDataSetter.EMPTY) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 4edcf871f6b..a58201f833f 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.21'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.26'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), @@ -9,6 +9,12 @@ ext { '5.6': properties.getOrDefault('neo4jVersion56', '5.6.0'), '5.7': properties.getOrDefault('neo4jVersion57', '5.7.0'), '5.8': properties.getOrDefault('neo4jVersion58', '5.8.0'), + '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), + '5.10': properties.getOrDefault('neo4jVersion510', '5.10.0'), + '5.11': properties.getOrDefault('neo4jVersion511', '5.11.0'), + '5.12': properties.getOrDefault('neo4jVersion512', '5.12.0'), + '5.13': properties.getOrDefault('neo4jVersion513', '5.13.0'), + '5.13': properties.getOrDefault('neo4jVersion513', '5.13.0'), ] neo4jDefault = neos.'4.4' @@ -27,6 +33,10 @@ ext { '5.7': '2.13.10', '5.8': '2.13.10', '5.9': '2.13.10', + '5.10': '2.13.10', + '5.11': '2.13.10', + '5.12': '2.13.10', + '5.13': '2.13.11', ] log4js = [ @@ -40,6 +50,10 @@ ext { '5.7': '2.20.0', '5.8': '2.20.0', '5.9': '2.20.0', + '5.10': '2.20.0', + '5.11': '2.20.0', + '5.12': '2.20.0', + '5.13': '2.20.0', ] ver = [ @@ -51,7 +65,7 @@ ext { 'auto-common' : '1.2.1', 'auto-service' : '1.0.1', 'bouncycastle-prov' : '1.72', - 'checkStyle' : '10.3.4', + 'checkStyle' : '10.12.1', 'commons-io' : '2.11.0', 'commons-lang3' : '3.12.0', 'commons-math3' : '3.6.1', diff --git a/gradle/version.gradle b/gradle/version.gradle index d2069cb5395..8ea3203fbae 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,3 +1,6 @@ ext { - gdsVersion = '2.4.0' + gdsBaseVersion = '2.4.7' + gdsAuraVersion = '37' + + gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index c1962a79e29..033e24c4cdf 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a21c6ebe28b..c747538fb38 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-all.zip networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index aeb74cbb43e..fcb6fca147c 100755 --- a/gradlew +++ b/gradlew @@ -130,10 +130,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMapping.java b/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMapping.java index a021bf97d1a..4cc2c2f1409 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMapping.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMapping.java @@ -67,6 +67,13 @@ public void validateProperties() { if (neoPropertyKey().equals(ElementProjection.PROJECT_ALL) && aggregation() != Aggregation.COUNT) { throw new IllegalArgumentException("A '*' property key can only be used in combination with count aggregation."); } + validatePropertyKey(propertyKey()); + } + + public static void validatePropertyKey(String propertyKey) { + if (propertyKey.isEmpty()) { + throw new IllegalArgumentException("Property key must not be empty."); + } } public static PropertyMapping fromObject(String propertyKey, Object stringOrMap) { diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java index f954b848ea4..4f358aa6cac 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java @@ -241,7 +241,6 @@ public long[] longArrayValue() { return defaultValue; } - @Override public String toString() { return "DefaultValue(" + defaultValue + ')'; diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValueUtil.java b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValueUtil.java index 13cb10ba70c..5fa72de2678 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValueUtil.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValueUtil.java @@ -27,7 +27,7 @@ import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -public final class DefaultValueUtil { +final class DefaultValueUtil { private DefaultValueUtil() {} @@ -50,7 +50,9 @@ static Object transformObjectToPrimitiveArray(Object[] defaultArrayValue) { static double[] parseDoubleArrayValue(Object defaultValue, ValueType type) { double[] defaultDoubleArray; - if (defaultValue instanceof Collection) { + if (defaultValue instanceof double[]) { + defaultDoubleArray = (double[]) defaultValue; + } else if (defaultValue instanceof Collection) { defaultDoubleArray = ((Collection) defaultValue).stream() .map(Object::toString) .mapToDouble(Double::parseDouble) @@ -73,7 +75,9 @@ static double[] parseDoubleArrayValue(Object defaultValue, ValueType type) { static long[] parseLongArrayValue(Object defaultValue, ValueType type) { long[] defaultLongArray; - if (defaultValue instanceof Collection) { + if (defaultValue instanceof long[]) { + defaultLongArray = (long[]) defaultValue; + } else if (defaultValue instanceof Collection) { defaultLongArray = ((Collection) defaultValue).stream() .map(Object::toString) .mapToLong(Long::parseLong) @@ -96,7 +100,9 @@ static long[] parseLongArrayValue(Object defaultValue, ValueType type) { static float[] parseFloatArrayValue(Object defaultValue, ValueType type) { float[] defaultFloatArray; - if (defaultValue instanceof List) { + if (defaultValue instanceof float[]) { + defaultFloatArray = (float[]) defaultValue; + } else if (defaultValue instanceof List) { var df = ((List) defaultValue); defaultFloatArray = new float[df.size()]; for (int i = 0; i < df.size(); i++) { diff --git a/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java b/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java index 7d461186011..6e29abc1dea 100644 --- a/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java +++ b/graph-projection-api/src/test/java/org/neo4j/gds/PropertyMappingTest.java @@ -25,10 +25,8 @@ import java.util.Map; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; class PropertyMappingTest { @@ -88,13 +86,21 @@ void shouldSupportCaseInsensitiveConfigKeys() { @Test void failsOnWrongKeyType() { - IllegalArgumentException ex = assertThrows( - IllegalArgumentException.class, () -> PropertyMapping.fromObject("transaction_count", Map.of( + assertThatThrownBy( + () -> PropertyMapping.fromObject("transaction_count", Map.of( "property", 42 - ))); - assertThat( - ex.getMessage(), - containsString("Expected the value of 'property' to be of type String, but was 'Integer'.") - ); + ))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Expected the value of 'property' to be of type String, but was 'Integer'."); + } + + @Test + void failsOnEmptyPropertyKey() { + assertThatThrownBy( + () -> PropertyMapping.fromObject("", Map.of( + "neoKey", 42 + ))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Property key must not be empty"); } } diff --git a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java index 00ed2f9cc66..c7836c479df 100644 --- a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java +++ b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java @@ -30,7 +30,7 @@ import org.neo4j.gds.beta.filter.expression.EvaluationContext; import org.neo4j.gds.beta.filter.expression.Expression; import org.neo4j.gds.config.GraphSampleAlgoConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.ImmutableNodes; @@ -108,7 +108,7 @@ public double evaluate(EvaluationContext context) { idMap, config.concurrency(), Map.of(), - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java index 7d199c705a0..11477e04f27 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporter.java @@ -40,6 +40,8 @@ public abstract class GraphStoreExporter> neoNodeProperties; + private final Optional nodeLabelMapping; + public enum IdMappingType implements IdMapFunction { MAPPED { @Override @@ -84,13 +86,15 @@ interface IdMapFunction { protected GraphStoreExporter( GraphStore graphStore, CONFIG config, - Optional neoNodeProperties + Optional neoNodeProperties, + Optional nodeLabelMapping ) { this.graphStore = graphStore; this.config = config; this.neoNodeProperties = neoNodeProperties .map(NeoNodeProperties::neoNodeProperties) .orElse(Map.of()); + this.nodeLabelMapping = nodeLabelMapping; } protected abstract void export(GraphStoreInput graphStoreInput); @@ -99,7 +103,11 @@ protected GraphStoreExporter( public ExportedProperties run() { var metaDataStore = MetaDataStore.of(graphStore); - var nodeStore = NodeStore.of(graphStore, neoNodeProperties); + var nodeStore = NodeStore.of( + graphStore, + neoNodeProperties, + nodeLabelMapping + ); var relationshipStore = RelationshipStore.of(graphStore, config.defaultRelationshipType()); var graphProperties = graphStore .graphPropertyKeys() diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java index 7a74e29dbc8..4cb59a1e60e 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java @@ -48,6 +48,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Queue; import java.util.Set; import java.util.Spliterator; @@ -219,6 +220,10 @@ public InputIterable graphProperties() { return () -> new GraphPropertyIterator(graphProperties.iterator(), concurrency); } + public Optional labelMapping() { + return nodeStore.labelMapping(); + } + static class GraphPropertyIterator implements InputIterator { private final Iterator graphPropertyIterator; @@ -468,7 +473,7 @@ public boolean next(InputEntityVisitor visitor) throws IOException { if (hasProperties) { for (var label : labels) { - nodeStore.nodeProperties + nodeStore.labelToNodeProperties() .getOrDefault(label, Map.of()) .forEach((propertyKey, properties) -> exportProperty( visitor, @@ -478,7 +483,7 @@ public boolean next(InputEntityVisitor visitor) throws IOException { } } } else if (hasProperties) { // no label information, but node properties - nodeStore.nodeProperties.forEach((label, nodeProperties) -> nodeProperties.forEach((propertyKey, properties) -> exportProperty( + nodeStore.labelToNodeProperties().forEach((label, nodeProperties) -> nodeProperties.forEach((propertyKey, properties) -> exportProperty( visitor, propertyKey, properties::getObject diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java b/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java index 1d96439bf3e..13646fa18a2 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/MetaDataStore.java @@ -21,12 +21,14 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.io.file.GraphInfo; import org.neo4j.gds.core.io.file.ImmutableGraphInfo; +import java.util.Locale; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -43,8 +45,21 @@ static MetaDataStore of(GraphStore graphStore) { Function.identity(), graphStore::relationshipCount )); + + var idMapTypeId = graphStore.nodes().typeId(); + + if (idMapTypeId.equals(IdMap.NO_TYPE)) { + throw new IllegalArgumentException(String.format( + Locale.US, + "Cannot write graph store with untyped id map. Got instance of `%s`", + graphStore.nodes() + .getClass() + .getSimpleName())); + } + GraphInfo graphInfo = ImmutableGraphInfo.of( graphStore.databaseId(), + idMapTypeId, graphStore.nodeCount(), graphStore.nodes().highestOriginalId(), relTypeCounts, diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java new file mode 100644 index 00000000000..6712b7da489 --- /dev/null +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeLabelMapping.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io; + +import org.apache.commons.lang3.mutable.MutableInt; +import org.neo4j.gds.NodeLabel; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class NodeLabelMapping { + private final HashMap map; + private final HashMap inverseMap; + + public NodeLabelMapping(Collection labels) { + MutableInt index = new MutableInt(); + map = new HashMap<>(labels.stream() + .collect(Collectors.toMap(label -> label, label -> Integer.toString(index.getAndIncrement())))); + index.setValue(0); + inverseMap = new HashMap<>(labels.stream() + .collect(Collectors.toMap(label -> Integer.toString(index.getAndIncrement()), label -> label))); + } + + public String get(NodeLabel label) { + return map.get(label); + } + + public NodeLabel get(String labelName) { + return inverseMap.get(labelName); + } + + public Set> entrySet() { + return map.entrySet(); + } +} diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java b/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java index bf75afe7c02..97957d71cde 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/NodeStore.java @@ -27,8 +27,11 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.function.LongFunction; +import java.util.stream.Collectors; public class NodeStore { @@ -40,20 +43,25 @@ public class NodeStore { final IdMap idMap; - final Map> nodeProperties; + private final Map> nodeProperties; + final Map> additionalProperties; private final Set availableNodeLabels; private final boolean hasLabels; + private final Optional nodeLabelMapping; + private final Function labelNameFunction; + private NodeStore( long nodeCount, HugeIntArray labelCounts, IdMap idMap, boolean hasLabels, Map> nodeProperties, - Map> additionalProperties + Map> additionalProperties, + Optional nodeLabelMapping ) { this.nodeCount = nodeCount; this.labelCounts = labelCounts; @@ -62,6 +70,10 @@ private NodeStore( this.hasLabels = hasLabels; this.availableNodeLabels = idMap.availableNodeLabels(); this.additionalProperties = additionalProperties; + this.nodeLabelMapping = nodeLabelMapping; + this.labelNameFunction = nodeLabelMapping.isPresent() + ? nodeLabelMapping.get()::get + : (NodeLabel nodeLabel) -> nodeLabel.name(); } boolean hasLabels() { @@ -72,10 +84,25 @@ boolean hasProperties() { return nodeProperties != null; } + Map> labelToNodeProperties() { + return nodeLabelMapping.isPresent() + ? nodeProperties.entrySet() + .stream() + .collect(Collectors.toMap( + entry -> nodeLabelMapping.get().get(NodeLabel.of(entry.getKey())), + entry -> entry.getValue() + )) + : nodeProperties; + } + int labelCount() { return !hasLabels() ? 0 : idMap.availableNodeLabels().size(); } + Optional labelMapping() { + return nodeLabelMapping; + } + int propertyCount() { if (nodeProperties == null) { return 0; @@ -94,7 +121,7 @@ String[] labels(long nodeId) { int i = 0; for (var nodeLabel : availableNodeLabels) { if (idMap.hasLabel(nodeId, nodeLabel)) { - labels[i++] = nodeLabel.name; + labels[i++] = labelNameFunction.apply(nodeLabel); } } @@ -103,7 +130,8 @@ String[] labels(long nodeId) { static NodeStore of( GraphStore graphStore, - Map> additionalProperties + Map> additionalProperties, + Optional nodeLabelMapping ) { HugeIntArray labelCounts = null; @@ -137,7 +165,8 @@ static NodeStore of( nodeLabels, hasNodeLabels, nodeProperties.isEmpty() ? null : nodeProperties, - additionalProperties + additionalProperties, + nodeLabelMapping ); } } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/GdsParallelBatchImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/GdsParallelBatchImporter.java index 2b1c43d8c08..e1b7e89a68c 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/GdsParallelBatchImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/GdsParallelBatchImporter.java @@ -51,7 +51,6 @@ import static org.neo4j.configuration.GraphDatabaseSettings.SYSTEM_DATABASE_NAME; import static org.neo4j.gds.core.io.GraphStoreExporter.DIRECTORY_IS_WRITABLE; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -import static org.neo4j.internal.batchimport.input.BadCollector.UNLIMITED_TOLERANCE; public final class GdsParallelBatchImporter { @@ -177,10 +176,12 @@ public void writeDatabase(CompatInput compatInput, boolean startDatabase) { if (startDatabase) { var dbStartTimer = ProgressTimer.start(); if (createAndStartDatabase()) { - log.info(formatWithLocale( - "Database created and started after %s ms", - dbStartTimer.stop().getDuration() - )); + log.info( + formatWithLocale( + "Database created and started after %s ms", + dbStartTimer.stop().getDuration() + ) + ); } else { log.error("Unable to start database " + config.dbName()); } @@ -201,22 +202,22 @@ private void validateDatabaseDoesNotExist(GdsDatabaseLayout databaseLayout) { var metaDataPath = databaseLayout.metadataStore(); var dbExists = Files.exists(metaDataPath) && Files.isReadable(metaDataPath); if (dbExists && !config.force()) { - throw new IllegalArgumentException(formatWithLocale( - "The database [%s] already exists. The graph export procedure can only create new databases.", - config.dbName() - )); + throw new IllegalArgumentException( + formatWithLocale( + "The database [%s] already exists. The graph export procedure can only create new databases.", + config.dbName() + ) + ); } } - private LogService getLogService() { - return config.enableDebugLog() - ? logService - : NullLogService.getInstance(); - } + private LogService getLogService() { return config.enableDebugLog() + ? logService + : NullLogService.getInstance(); } private Collector getCollector() { return config.useBadCollector() - ? Collectors.badCollector(new LoggingOutputStream(log), UNLIMITED_TOLERANCE) + ? Collectors.badCollector(new LoggingOutputStream(log), 0) : Collector.EMPTY; } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java index a4b3e28128a..9d62cb0968f 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporter.java @@ -64,7 +64,7 @@ private GraphStoreToDatabaseExporter( Log log, ProgressTracker progressTracker ) { - super(graphStore, config, neoNodeProperties); + super(graphStore, config, neoNodeProperties, Optional.empty()); var executionMonitor = ProgressTrackerExecutionMonitor.of( progressTracker, ClockService.clock(), diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java index 56fbfccccf8..e3ab725380f 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java @@ -25,63 +25,27 @@ import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.Tasks; -import org.neo4j.internal.batchimport.CountGroupsStage; -import org.neo4j.internal.batchimport.DataImporter; -import org.neo4j.internal.batchimport.NodeCountsAndLabelIndexBuildStage; -import org.neo4j.internal.batchimport.NodeDegreeCountStage; -import org.neo4j.internal.batchimport.NodeFirstGroupStage; -import org.neo4j.internal.batchimport.RelationshipCountsAndTypeIndexBuildStage; -import org.neo4j.internal.batchimport.RelationshipGroupStage; -import org.neo4j.internal.batchimport.RelationshipLinkbackStage; -import org.neo4j.internal.batchimport.RelationshipLinkforwardStage; -import org.neo4j.internal.batchimport.ScanAndCacheGroupsStage; -import org.neo4j.internal.batchimport.SparseNodeFirstRelationshipStage; -import org.neo4j.internal.batchimport.WriteGroupsStage; import org.neo4j.internal.batchimport.staging.ExecutionMonitor; import org.neo4j.internal.batchimport.staging.StageExecution; -import org.neo4j.internal.batchimport.stats.Keys; -import org.neo4j.internal.batchimport.stats.Stat; -import org.neo4j.internal.batchimport.store.BatchingNeoStores; -import org.neo4j.internal.helpers.collection.Iterables; import java.time.Clock; -import java.util.Objects; +import java.util.List; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.StreamSupport; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public final class ProgressTrackerExecutionMonitor implements CompatExecutionMonitor { private final Clock clock; private final long intervalMillis; - // The total task volume per stage. - // Set to 0 at the beginning of a stage. - private final AtomicLong stageProgressTotal; - // The progressed task volume per stage. - // Set to 0 at the beginning of a stage. - private final AtomicLong stageProgressCurrent; - private final ProgressTracker progressTracker; - private DependencyResolver dependencyResolver; - public static Task progressTask(long nodeCount, long relationshipCount) { + public static Task progressTask() { return Tasks.task( GraphStoreToDatabaseExporter.class.getSimpleName(), - Tasks.leaf(DataImporter.NODE_IMPORT_NAME, nodeCount), - Tasks.leaf(DataImporter.RELATIONSHIP_IMPORT_NAME, relationshipCount), - Tasks.leaf(NodeDegreeCountStage.NAME), - Tasks.leaf(RelationshipLinkforwardStage.NAME, 2 * relationshipCount), - Tasks.leaf(RelationshipGroupStage.NAME), - Tasks.leaf(SparseNodeFirstRelationshipStage.NAME), - Tasks.leaf(RelationshipLinkbackStage.NAME, 2 * relationshipCount), - Tasks.leaf(CountGroupsStage.NAME), - Tasks.leaf(ScanAndCacheGroupsStage.NAME), - Tasks.leaf(WriteGroupsStage.NAME), - Tasks.leaf(NodeFirstGroupStage.NAME), - Tasks.leaf(NodeCountsAndLabelIndexBuildStage.NAME, nodeCount), - Tasks.leaf(RelationshipCountsAndTypeIndexBuildStage.NAME, relationshipCount) + List.of() ); } @@ -98,46 +62,31 @@ private ProgressTrackerExecutionMonitor(ProgressTracker progressTracker, Clock c this.clock = clock; this.intervalMillis = unit.toMillis(time); this.progressTracker = progressTracker; - this.stageProgressTotal = new AtomicLong(0); - this.stageProgressCurrent = new AtomicLong(0); } @Override public void initialize(DependencyResolver dependencyResolver) { - this.dependencyResolver = dependencyResolver; this.progressTracker.beginSubTask(); } @Override public void start(StageExecution execution) { - this.stageProgressTotal.set(0); - this.stageProgressCurrent.set(0); - - var neoStores = this.dependencyResolver.resolveDependency(BatchingNeoStores.class); - var relationshipRecordIdCount = Neo4jProxy.getHighId(neoStores.getRelationshipStore()); - var groupCount = Neo4jProxy.getHighId(neoStores.getTemporaryRelationshipGroupStore()); - - switch (execution.getStageName()) { - case NodeDegreeCountStage.NAME: - this.progressTracker.beginSubTask(execution.getStageName(), relationshipRecordIdCount); - break; - case CountGroupsStage.NAME: - case WriteGroupsStage.NAME: - case NodeFirstGroupStage.NAME: - this.progressTracker.beginSubTask(execution.getStageName(), groupCount); - break; - default: - this.progressTracker.beginSubTask(execution.getStageName()); - } - - if (this.progressTracker.currentVolume() != Task.UNKNOWN_VOLUME) { - this.stageProgressTotal.set(this.progressTracker.currentVolume()); - } + progressTracker.logInfo( + formatWithLocale( + "%s :: Start", + execution.getStageName() + ) + ); } @Override public void end(StageExecution execution, long totalTimeMillis) { - this.progressTracker.endSubTask(execution.getStageName()); + progressTracker.logInfo( + formatWithLocale( + "%s :: Finished", + execution.getStageName() + ) + ); } @Override @@ -148,23 +97,7 @@ public void done(boolean successful, long totalTimeMillis, String additionalInfo @Override public void check(StageExecution execution) { - // Cap the total to not produce percentages > 100. - var progress = Math.min(progress(execution), this.stageProgressTotal.get()); - this.progressTracker.logProgress(progress - this.stageProgressCurrent.getAndSet(progress)); - } - private static long progress(StageExecution execution) { - return StreamSupport - .stream(execution.steps().spliterator(), false) - .map(step -> step.stats().stat(Keys.progress)) - .filter(Objects::nonNull) - .map(Stat::asLong) - .findFirst() - .orElseGet(() -> { - var doneBatches = Iterables.last(execution.steps()).stats().stat(Keys.done_batches).asLong(); - var batchSize = execution.getConfig().batchSize(); - return doneBatches * batchSize; - }); } @Override diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/ElementVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/ElementVisitor.java index 6a29089eadf..dc513f170c3 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/ElementVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/ElementVisitor.java @@ -56,7 +56,7 @@ abstract class ElementVisitor extends In protected abstract List getPropertySchema(); - abstract void reset(); + public abstract void reset(); @Override public boolean property(String key, Object value) { diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java index 5122e3a5e50..78efdb0563e 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java @@ -26,13 +26,16 @@ import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.internal.batchimport.InputIterable; +import java.util.HashMap; import java.util.Map; +import java.util.Optional; public interface FileInput extends CompatInput { InputIterable graphProperties(); String userName(); GraphInfo graphInfo(); MutableNodeSchema nodeSchema(); + Optional> labelMapping(); MutableRelationshipSchema relationshipSchema(); Map graphPropertySchema(); Capabilities capabilities(); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 1c12cc39968..e33c3119a69 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -28,8 +28,8 @@ import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.schema.ImmutableMutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.io.GraphStoreGraphPropertyVisitor; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; import org.neo4j.gds.core.loading.Capabilities.WriteMode; @@ -150,12 +150,21 @@ private Nodes importNodes(FileInput fileInput) { progressTracker.beginSubTask(); MutableNodeSchema nodeSchema = fileInput.nodeSchema(); graphSchemaBuilder.nodeSchema(nodeSchema); + nodeSchema.entries().stream().forEach(entry -> log.info("Imported node label schema: %s", entry.identifier())); + var labelMapping = fileInput.labelMapping(); + if (labelMapping.isPresent()) { + labelMapping.get().entrySet().forEach(entry -> log.info("Label mapping: %s -> %s", entry.getKey(), entry.getValue())); + } + else { + log.info("Label mapping file was not found, continuing import without label mapping"); + } NodesBuilder nodesBuilder = GraphFactory.initNodesBuilder(nodeSchema) .maxOriginalId(fileInput.graphInfo().maxOriginalId()) .concurrency(concurrency) .nodeCount(fileInput.graphInfo().nodeCount()) .deduplicateIds(false) + .idMapBuilderType(fileInput.graphInfo().idMapBuilderType()) .build(); nodeVisitorBuilder.withNodeSchema(nodeSchema); nodeVisitorBuilder.withNodesBuilder(nodesBuilder); @@ -166,7 +175,7 @@ private Nodes importNodes(FileInput fileInput) { (index) -> new ElementImportRunner<>(nodeVisitorBuilder.build(), nodesIterator, progressTracker) ); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var nodes = nodesBuilder.build(); @@ -197,7 +206,7 @@ private void importRelationships(FileInput fileInput, IdMap nodes) { (index) -> new ElementImportRunner<>(relationshipVisitorBuilder.build(), relationshipsIterator, progressTracker) ); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var relationshipImportResult = relationshipImportResult(relationshipBuildersByType); @@ -226,7 +235,7 @@ private void importGraphProperties(FileInput fileInput) { progressTracker ) ); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); graphStoreGraphPropertyVisitor.close(); graphStoreBuilder.graphProperties(GraphPropertyStoreFromVisitorHelper.fromGraphPropertyVisitor( diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java index d2b7cc23611..96c9ea45d06 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphInfo.java @@ -29,6 +29,7 @@ @ValueClass public interface GraphInfo { DatabaseId databaseId(); + String idMapBuilderType(); long nodeCount(); long maxOriginalId(); Map relationshipTypeCounts(); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java index 1d854e994bd..c86360ae2f9 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporter.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.io.file; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.CompatInput; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -26,9 +27,11 @@ import org.neo4j.gds.core.io.GraphStoreExporter; import org.neo4j.gds.core.io.GraphStoreInput; import org.neo4j.gds.core.io.NeoNodeProperties; +import org.neo4j.gds.core.io.NodeLabelMapping; import org.neo4j.gds.core.io.schema.ElementSchemaVisitor; import org.neo4j.gds.core.io.schema.NodeSchemaVisitor; import org.neo4j.gds.core.io.schema.RelationshipSchemaVisitor; +import org.neo4j.gds.core.io.schema.SimpleVisitor; import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -41,6 +44,7 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -53,6 +57,8 @@ public class GraphStoreToFileExporter extends GraphStoreExporter> userInfoVisitorSupplier; private final Supplier> graphInfoVisitorSupplier; private final Supplier nodeSchemaVisitorSupplier; + private final Supplier>> labelMappingVisitorSupplier; + private final Supplier relationshipSchemaVisitorSupplier; private final Supplier graphPropertySchemaVisitorSupplier; private final Supplier> graphCapabilitiesWriterSupplier; @@ -65,9 +71,11 @@ public GraphStoreToFileExporter( GraphStore graphStore, GraphStoreToFileExporterConfig config, Optional neoNodeProperties, + Optional nodeLabelMapping, Supplier> userInfoVisitorSupplier, Supplier> graphInfoVisitorSupplier, Supplier nodeSchemaVisitorSupplier, + Supplier>> labelMappingVisitorSupplier, Supplier relationshipSchemaVisitorSupplier, Supplier graphPropertySchemaVisitorSupplier, Supplier> graphCapabilitiesWriterSupplier, @@ -78,13 +86,14 @@ public GraphStoreToFileExporter( Log log, String rootTaskName ) { - super(graphStore, config, neoNodeProperties); + super(graphStore, config, neoNodeProperties, nodeLabelMapping); this.nodeVisitorSupplier = nodeVisitorSupplier; this.relationshipVisitorSupplier = relationshipVisitorSupplier; this.graphPropertyVisitorSupplier = graphPropertyVisitorSupplier; this.userInfoVisitorSupplier = userInfoVisitorSupplier; this.graphInfoVisitorSupplier = graphInfoVisitorSupplier; this.nodeSchemaVisitorSupplier = nodeSchemaVisitorSupplier; + this.labelMappingVisitorSupplier = labelMappingVisitorSupplier; this.relationshipSchemaVisitorSupplier = relationshipSchemaVisitorSupplier; this.graphPropertySchemaVisitorSupplier = graphPropertySchemaVisitorSupplier; this.graphCapabilitiesWriterSupplier = graphCapabilitiesWriterSupplier; @@ -103,11 +112,20 @@ protected void export(GraphStoreInput graphStoreInput) { exportGraphPropertySchema(graphStoreInput); exportGraphCapabilities(graphStoreInput); } + exportNodeLabelMapping(graphStoreInput); + var progressTracker = createProgressTracker(graphStoreInput); - progressTracker.beginSubTask(); - exportNodes(graphStoreInput, progressTracker); - exportRelationships(graphStoreInput, progressTracker); - exportGraphProperties(graphStoreInput, progressTracker); + + try { + progressTracker.beginSubTask(); + exportNodes(graphStoreInput, progressTracker); + exportRelationships(graphStoreInput, progressTracker); + exportGraphProperties(graphStoreInput, progressTracker); + } catch (Exception e) { + // as the tracker is created in this method + progressTracker.endSubTaskWithFailure(); + throw e; + } progressTracker.endSubTask(); } @@ -239,6 +257,17 @@ private void exportNodeSchema(GraphStoreInput graphStoreInput) { } } + private void exportNodeLabelMapping(GraphStoreInput graphStoreInput) { + var labelMapping = graphStoreInput.labelMapping(); + if (labelMapping.isPresent()) { + try (var labelMappingVisitor = labelMappingVisitorSupplier.get()) { + labelMapping.get().entrySet().forEach(entry -> labelMappingVisitor.export(entry)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + } + private void exportRelationshipSchema(GraphStoreInput graphStoreInput) { var relationshipSchema = graphStoreInput.metaDataStore().relationshipSchema(); try (var relationshipSchemaVisitor = relationshipSchemaVisitorSupplier.get()) { diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java index 5fbc0e944f6..c1c4ead72db 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphStoreToFileExporterConfig.java @@ -42,6 +42,12 @@ default String username() { String exportName(); + @Value.Default + default boolean useLabelMapping() { + // the reason for default false (unlike BackupConfig) is to prevent a breaking change in CSV export + return false; + } + static GraphStoreToFileExporterConfig of(String username, CypherMapWrapper config) { return new GraphStoreToFileExporterConfigImpl(username, config); } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java index 7c97af5307e..2358255b49e 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/NodeVisitor.java @@ -35,11 +35,11 @@ public abstract class NodeVisitor extends ElementVisitor { private static final List EMPTY_LABELS = Collections.emptyList(); - private static final Set EMPTY_LABELS_LABEL = Set.of(NodeLabel.ALL_NODES); + protected static final Set EMPTY_LABELS_LABEL = Set.of(NodeLabel.ALL_NODES); - private final NodeSchema nodeSchema; + protected final NodeSchema nodeSchema; private long currentId; - private List currentLabels; + protected List currentLabels; private String labelIdentifier; protected NodeVisitor(NodeSchema nodeSchema) { @@ -106,7 +106,7 @@ protected List getPropertySchema() { } @Override - void reset() { + public void reset() { currentId = -1; currentLabels = EMPTY_LABELS; labelIdentifier = ""; diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/RelationshipVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/RelationshipVisitor.java index ce848e7e5b8..3ada3cdcd7d 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/RelationshipVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/RelationshipVisitor.java @@ -99,7 +99,7 @@ protected List getPropertySchema() { } @Override - void reset() { + public void reset() { currentStartNode = -1; currentEndNode = -1; } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java new file mode 100644 index 00000000000..8c4a56608f5 --- /dev/null +++ b/io/core/src/main/java/org/neo4j/gds/core/io/schema/SimpleVisitor.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.schema; + +import java.io.Closeable; + +public interface SimpleVisitor extends Closeable { + void export(T value); +} diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java index 4a4a776d3f1..b4ccab3764c 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.apache.commons.lang3.tuple.Pair; +import org.eclipse.collections.impl.block.factory.Functions; import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; @@ -52,8 +53,10 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; final class CsvFileInput implements FileInput { @@ -80,6 +83,7 @@ final class CsvFileInput implements FileInput { private final String userName; private final GraphInfo graphInfo; private final MutableNodeSchema nodeSchema; + private final Optional> labelMapping; private final MutableRelationshipSchema relationshipSchema; private final Map graphPropertySchema; private final Capabilities capabilities; @@ -89,6 +93,7 @@ final class CsvFileInput implements FileInput { this.userName = new UserInfoLoader(importPath).load(); this.graphInfo = new GraphInfoLoader(importPath, CSV_MAPPER).load(); this.nodeSchema = new NodeSchemaLoader(importPath).load(); + this.labelMapping = new NodeLabelMappingLoader(importPath).load(); this.relationshipSchema = new RelationshipSchemaLoader(importPath).load(); this.graphPropertySchema = new GraphPropertySchemaLoader(importPath).load(); this.capabilities = new GraphCapabilitiesLoader(importPath, CSV_MAPPER).load(); @@ -97,10 +102,15 @@ final class CsvFileInput implements FileInput { @Override public InputIterable nodes(Collector badCollector) { Map> pathMapping = CsvImportFileUtil.nodeHeaderToFileMapping(importPath); - Map> headerToDataFilesMapping = pathMapping.entrySet().stream().collect(Collectors.toMap( - entry -> CsvImportFileUtil.parseNodeHeader(entry.getKey()), - Map.Entry::getValue - )); + Map> headerToDataFilesMapping = pathMapping.entrySet() + .stream() + .collect(Collectors.toMap( + entry -> CsvImportFileUtil.parseNodeHeader( + entry.getKey(), + labelMapping.isPresent() ? labelMapping.get()::get : Functions.identity() + ), + Map.Entry::getValue + )); return () -> new NodeImporter(headerToDataFilesMapping, nodeSchema); } @@ -157,6 +167,11 @@ public MutableNodeSchema nodeSchema() { return nodeSchema; } + @Override + public Optional> labelMapping() { + return labelMapping; + } + @Override public MutableRelationshipSchema relationshipSchema() { return relationshipSchema; diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java index 24fb3f72e4f..799134da388 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitor.java @@ -33,6 +33,7 @@ public class CsvGraphInfoVisitor implements SingleRowVisitor { public static final String GRAPH_INFO_FILE_NAME = "graph_info.csv"; public static final String DATABASE_NAME_COLUMN_NAME = "databaseName"; + public static final String ID_MAP_BUILDER_TYPE_COLUMN_NAME = "idMapBuilderType"; public static final String NODE_COUNT_COLUMN_NAME = "nodeCount"; public static final String MAX_ORIGINAL_ID_COLUMN_NAME = "maxOriginalId"; public static final String REL_TYPE_COUNTS_COLUMN_NAME = "relTypeCounts"; @@ -61,6 +62,7 @@ public void export(GraphInfo graphInfo) { this.csvWriter.writeRow( graphInfo.databaseId().databaseName(), + graphInfo.idMapBuilderType(), Long.toString(graphInfo.nodeCount()), Long.toString(graphInfo.maxOriginalId()), CsvMapUtil.relationshipCountsToString(graphInfo.relationshipTypeCounts()), @@ -80,6 +82,7 @@ public void close() { private void writeHeader() { this.csvWriter.writeRow( DATABASE_NAME_COLUMN_NAME, + ID_MAP_BUILDER_TYPE_COLUMN_NAME, NODE_COUNT_COLUMN_NAME, MAX_ORIGINAL_ID_COLUMN_NAME, REL_TYPE_COUNTS_COLUMN_NAME, diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtil.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtil.java index 60505e61dcf..47bc43eb9d2 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtil.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtil.java @@ -30,6 +30,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -44,13 +45,18 @@ public final class CsvImportFileUtil { private CsvImportFileUtil() {} - public static NodeFileHeader parseNodeHeader(Path headerFile) { + public static NodeFileHeader parseNodeHeader(Path headerFile, Function labelMapping) { try (MappingIterator iterator = HEADER_FILE_READER.readValues(headerFile.toFile())) { var headerLine = iterator.next(); if (headerLine == null) { throw new UncheckedIOException(new IOException("Header line was null")); } - return NodeFileHeader.of(headerLine, inferNodeLabels(headerFile)); + return NodeFileHeader.of( + headerLine, + Arrays.stream(inferNodeLabels(headerFile)) + .map(label -> labelMapping.apply(label)) + .toArray(String[]::new) + ); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -107,7 +113,10 @@ static List getGraphPropertyHeaderFiles(Path csvDirectory) { return getFilesByRegex(csvDirectory, graphPropertyFilesPattern); } - private static Map> headerToFileMapping(Path csvDirectory, Function> headerPaths) { + private static Map> headerToFileMapping( + Path csvDirectory, + Function> headerPaths + ) { Map> headerToDataFileMapping = new HashMap<>(); for (Path headerFile : headerPaths.apply(csvDirectory)) { String dataFilePattern = headerFile.getFileName().toString().replace("_header", "(_\\d+)"); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvMapUtil.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvMapUtil.java index 101d1f03684..6e09c01c38d 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvMapUtil.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvMapUtil.java @@ -42,6 +42,10 @@ static Map fromString( Function keyParser, Function valueParser ) { + if (mapString.isEmpty()) { + return Map.of(); + } + var listElements = mapString.split(String.valueOf(LIST_DELIMITER)); var map = new HashMap(); for (int i = 0; i < listElements.length; i+=2) { diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java new file mode 100644 index 00000000000..18303b77891 --- /dev/null +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeLabelMappingVisitor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.file.csv; + +import de.siegmar.fastcsv.writer.CsvWriter; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.core.io.schema.SimpleVisitor; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Map; + +public class CsvNodeLabelMappingVisitor implements SimpleVisitor> { + + private static final String LABEL_MAPPING = "index"; + private static final String LABEL_COLUMN_NAME = "label"; + static final String LABEL_MAPPING_FILE_NAME = "label-mappings.csv"; + private final CsvWriter csvWriter; + + CsvNodeLabelMappingVisitor(Path fileLocation) { + try { + this.csvWriter = CsvWriter.builder().build(fileLocation.resolve(LABEL_MAPPING_FILE_NAME), StandardCharsets.UTF_8); + writeHeader(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void export(Map.Entry nodeLabelMapping) { + var row = new ArrayList(); + row.add(nodeLabelMapping.getValue()); + row.add(nodeLabelMapping.getKey().name()); + csvWriter.writeRow(row); + } + + private void writeHeader() { + csvWriter.writeRow( + LABEL_MAPPING, + LABEL_COLUMN_NAME + ); + } + + @Override + public void close() throws IOException { + try { + csvWriter.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitor.java index 358b8c22d3f..e464d7aaf1b 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitor.java @@ -55,7 +55,7 @@ protected void export() { if (key() != null) { row.add(key()); row.add(valueType().csvName()); - row.add(defaultValue().toString()); + row.add(DefaultValueIOHelper.serialize(defaultValue())); row.add(state().name()); } csvWriter.writeRow(row); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitor.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitor.java index 009d380f082..505dc7e71b9 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitor.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitor.java @@ -21,18 +21,24 @@ import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.jetbrains.annotations.TestOnly; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.api.schema.PropertySchema; +import org.neo4j.gds.core.io.NodeLabelMapping; import org.neo4j.gds.core.io.file.NodeVisitor; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -45,22 +51,26 @@ public class CsvNodeVisitor extends NodeVisitor { private final Map csvAppenders; private final Set headerFiles; + private final Optional nodeLabelMapping; + CsvNodeVisitor( Path fileLocation, NodeSchema nodeSchema, Set headerFiles, - int visitorId + int visitorId, + Optional nodeLabelMapping ) { super(nodeSchema); this.fileLocation = fileLocation; this.headerFiles = headerFiles; this.visitorId = visitorId; + this.nodeLabelMapping = nodeLabelMapping; this.csvAppenders = new HashMap<>(); } @TestOnly public CsvNodeVisitor(Path fileLocation, NodeSchema nodeSchema) { - this(fileLocation, nodeSchema, new HashSet<>(), 0); + this(fileLocation, nodeSchema, new HashSet<>(), 0, Optional.empty()); } @Override @@ -160,4 +170,18 @@ private JacksonFileAppender fileAppender(Path filePath, UnaryOperator getPropertySchema() { + var nodeLabelList = currentLabels.isEmpty() + ? EMPTY_LABELS_LABEL + : currentLabels.stream() + .map(nodeLabelMapping.isPresent() + ? nodeLabelMapping.get()::get + : NodeLabel::of + ) + .collect(Collectors.toSet()); + var propertySchemaForLabels = nodeSchema.filter(nodeLabelList); + return new ArrayList<>(propertySchemaForLabels.unionProperties().values()); + } } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java new file mode 100644 index 00000000000..b3fd00d2b57 --- /dev/null +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelper.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.file.csv; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +final class DefaultValueIOHelper { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String DEFAULT_VALUE_TEMPLATE = "DefaultValue(%s)"; + + private DefaultValueIOHelper() {} + + static String serialize(DefaultValue defaultValue) { + try { + var serializedValue = OBJECT_MAPPER.writeValueAsString(defaultValue.getObject()); + return formatWithLocale(DEFAULT_VALUE_TEMPLATE, serializedValue); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + static DefaultValue deserialize(String serializedValue, ValueType valueType, boolean isUserDefined) { + try { + if (serializedValue == null) { + return valueType.fallbackValue(); + } + + var value = serializedValue.replaceAll("DefaultValue\\(|null|NaN|\\)", ""); + if (value.isEmpty()) { + return valueType.fallbackValue(); + } + + Object parseValue; + switch (valueType) { + case DOUBLE: + parseValue = OBJECT_MAPPER.readValue(value, double.class); + break; + case LONG: + parseValue = OBJECT_MAPPER.readValue(value, long.class); + break; + case LONG_ARRAY: + parseValue = OBJECT_MAPPER.readValue(value, long[].class); + break; + case FLOAT_ARRAY: + parseValue = OBJECT_MAPPER.readValue(value, float[].class); + break; + case DOUBLE_ARRAY: + parseValue = OBJECT_MAPPER.readValue(value, double[].class); + break; + default: + throw new IllegalArgumentException("Cannot deserialize type `" + valueType + "` to DefaultValue"); + } + return DefaultValue.of(parseValue, valueType, isUserDefined); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } +} diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java index 94554de100b..2191cd98366 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoader.java @@ -31,6 +31,7 @@ import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DatabaseId; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.io.file.GraphInfo; import org.neo4j.gds.core.io.file.ImmutableGraphInfo; @@ -65,12 +66,12 @@ public GraphInfo load() { var databaseId = DatabaseId.from(line.databaseName); return ImmutableGraphInfo.builder() .databaseId(databaseId) + .idMapBuilderType(line.idMapBuilderType) .nodeCount(line.nodeCount) .maxOriginalId(line.maxOriginalId) .relationshipTypeCounts(line.relTypeCounts) .inverseIndexedRelationshipTypes(line.inverseIndexedRelTypes) .build(); - } catch (IOException e) { throw new UncheckedIOException(e); } @@ -85,6 +86,9 @@ public static class GraphInfoLine { @JsonProperty String databaseName; + @JsonProperty + String idMapBuilderType = IdMap.NO_TYPE; + @JsonProperty long nodeCount; diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphPropertySchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphPropertySchemaLoader.java index 82a8f6ff7ef..e3b661a7dbc 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphPropertySchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphPropertySchemaLoader.java @@ -25,7 +25,6 @@ import com.fasterxml.jackson.dataformat.csv.CsvMapper; import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.PropertySchema; @@ -61,7 +60,7 @@ Map load() { var schemaLine = linesIterator.next(); schemaBuilder.key(schemaLine.propertyKey); schemaBuilder.valueType(schemaLine.valueType); - schemaBuilder.defaultValue(DefaultValue.of(schemaLine.defaultValue, schemaLine.valueType, true)); + schemaBuilder.defaultValue(DefaultValueIOHelper.deserialize(schemaLine.defaultValue, schemaLine.valueType, true)); schemaBuilder.state(schemaLine.state); schemaBuilder.endOfEntity(); @@ -84,7 +83,6 @@ public static class PropertySchemaLine { ValueType valueType; @JsonProperty - @JsonDeserialize(converter = JacksonConverters.DefaultValueConverter.class) String defaultValue; @JsonProperty diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java index e606f114b9f..6e94b7f086d 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java @@ -24,6 +24,7 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.io.NeoNodeProperties; +import org.neo4j.gds.core.io.NodeLabelMapping; import org.neo4j.gds.core.io.file.GraphStoreToFileExporter; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfig; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; @@ -69,13 +70,19 @@ public static GraphStoreToFileExporter create( .forEach(label -> neoNodeSchema.getOrCreateLabel(label).addProperty(key, ValueType.STRING)) )); + Optional nodeLabelMapping = config.useLabelMapping() + ? Optional.of(new NodeLabelMapping(graphStore.nodeLabels())) + : Optional.empty(); + return new GraphStoreToFileExporter( graphStore, config, neoNodeProperties, - () -> new UserInfoVisitor(exportPath), + nodeLabelMapping, + () -> new UserInfoVisitor(exportPath), () -> new CsvGraphInfoVisitor(exportPath), () -> new CsvNodeSchemaVisitor(exportPath), + () -> new CsvNodeLabelMappingVisitor(exportPath), () -> new CsvRelationshipSchemaVisitor(exportPath), () -> new CsvGraphPropertySchemaVisitor(exportPath), () -> new CsvGraphCapabilitiesWriter(exportPath), @@ -83,7 +90,8 @@ public static GraphStoreToFileExporter create( exportPath, nodeSchema.union(neoNodeSchema), headerFiles, - index + index, + nodeLabelMapping ), (index) -> new CsvRelationshipVisitor(exportPath, relationshipSchema, headerFiles, index), (index) -> new CsvGraphPropertyVisitor( diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/JacksonConverters.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/JacksonConverters.java index a3865bead2b..88e3a312e03 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/JacksonConverters.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/JacksonConverters.java @@ -46,11 +46,4 @@ public ValueType convert(String value) { return ValueType.fromCsvName(value); } } - - static class DefaultValueConverter extends StdConverter { - @Override - public String convert(String value) { - return value.replaceAll("DefaultValue\\(|null|NaN|\\)", ""); - } - } } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java new file mode 100644 index 00000000000..da7a8f55763 --- /dev/null +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeLabelMappingLoader.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.file.csv; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvParser; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Optional; + +public class NodeLabelMappingLoader { + + private final ObjectReader objectReader; + private final Path labelMappingPath; + private final HashMap mapping; + + NodeLabelMappingLoader(Path csvDirectory) { + this.mapping = new HashMap(); + this.labelMappingPath = csvDirectory.resolve(CsvNodeLabelMappingVisitor.LABEL_MAPPING_FILE_NAME); + CsvMapper csvMapper = new CsvMapper(); + csvMapper.enable(CsvParser.Feature.TRIM_SPACES); + CsvSchema schema = CsvSchema.emptySchema().withHeader(); + this.objectReader = csvMapper.readerFor(MappingLine.class).with(schema); + } + + Optional> load() { + var file = labelMappingPath.toFile(); + if (!file.isFile()) { + return Optional.empty(); + } + + try(var reader = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { + var linesIterator = objectReader.readValues(reader); + while(linesIterator.hasNext()) { + var mappingLine = linesIterator.next(); + mapping.put(mappingLine.index, mappingLine.label); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return Optional.of(mapping); + } + + public static class MappingLine { + @JsonProperty + String index; + + @JsonProperty + String label; + } +} diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java index 18fbfee3dfc..6ea5d97add0 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java @@ -26,7 +26,6 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.MutableNodeSchema; @@ -62,7 +61,7 @@ MutableNodeSchema load() { if (schemaLine.propertyKey != null) { schemaBuilder.key(schemaLine.propertyKey); schemaBuilder.valueType(schemaLine.valueType); - schemaBuilder.defaultValue(DefaultValue.of(schemaLine.defaultValue, schemaLine.valueType, true)); + schemaBuilder.defaultValue(DefaultValueIOHelper.deserialize(schemaLine.defaultValue, schemaLine.valueType, true)); schemaBuilder.state(schemaLine.state); } schemaBuilder.endOfEntity(); @@ -89,7 +88,6 @@ public static class SchemaLine { ValueType valueType; @JsonProperty - @JsonDeserialize(converter = JacksonConverters.DefaultValueConverter.class) String defaultValue; @JsonProperty diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java index d2802af58aa..89fff99de74 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java @@ -26,7 +26,6 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; @@ -65,7 +64,7 @@ MutableRelationshipSchema load() { if (schemaLine.propertyKey != null) { schemaBuilder.key(schemaLine.propertyKey); schemaBuilder.valueType(schemaLine.valueType); - schemaBuilder.defaultValue(DefaultValue.of(schemaLine.defaultValue, schemaLine.valueType, true)); + schemaBuilder.defaultValue(DefaultValueIOHelper.deserialize(schemaLine.defaultValue, schemaLine.valueType, true)); schemaBuilder.state(schemaLine.state); schemaBuilder.aggregation(schemaLine.aggregation); } @@ -96,7 +95,6 @@ public static class SchemaLine { ValueType valueType; @JsonProperty - @JsonDeserialize(converter = JacksonConverters.DefaultValueConverter.class) String defaultValue; @JsonProperty diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java index 26b326ab281..e4001d7d2f1 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvGraphInfoVisitorTest.java @@ -36,10 +36,19 @@ class CsvGraphInfoVisitorTest extends CsvVisitorTest { @Test void shouldExportGraphInfo() { DatabaseId databaseId = DatabaseId.random(); + String idMapBuilderType = "custom"; CsvGraphInfoVisitor graphInfoVisitor = new CsvGraphInfoVisitor(tempDir); var relationshipTypeCounts = Map.of(RelationshipType.of("REL1"), 42L, RelationshipType.of("REL2"), 1337L); var inverseIndexedRelTypes = List.of(RelationshipType.of("REL1"),RelationshipType.of("REL2")); - graphInfoVisitor.export(ImmutableGraphInfo.of(databaseId, 1337L, 19L, relationshipTypeCounts, inverseIndexedRelTypes)); + var graphInfo = ImmutableGraphInfo.builder() + .databaseId(databaseId) + .idMapBuilderType(idMapBuilderType) + .nodeCount(1337L) + .maxOriginalId(19L) + .relationshipTypeCounts(relationshipTypeCounts) + .inverseIndexedRelationshipTypes(inverseIndexedRelTypes) + .build(); + graphInfoVisitor.export(graphInfo); graphInfoVisitor.close(); assertCsvFiles(List.of(GRAPH_INFO_FILE_NAME)); @@ -49,6 +58,7 @@ void shouldExportGraphInfo() { defaultHeaderColumns(), List.of( databaseId.databaseName(), + idMapBuilderType, Long.toString(1337L), Long.toString(19L), CsvMapUtil.relationshipCountsToString(relationshipTypeCounts), @@ -62,6 +72,7 @@ void shouldExportGraphInfo() { protected List defaultHeaderColumns() { return List.of( CsvGraphInfoVisitor.DATABASE_NAME_COLUMN_NAME, + CsvGraphInfoVisitor.ID_MAP_BUILDER_TYPE_COLUMN_NAME, CsvGraphInfoVisitor.NODE_COUNT_COLUMN_NAME, CsvGraphInfoVisitor.MAX_ORIGINAL_ID_COLUMN_NAME, CsvGraphInfoVisitor.REL_TYPE_COUNTS_COLUMN_NAME, diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java index 9649c149e6f..7d78a5bf788 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvImportFileUtilTest.java @@ -20,6 +20,7 @@ package org.neo4j.gds.core.io.file.csv; import org.apache.commons.io.FileUtils; +import org.eclipse.collections.impl.block.factory.Functions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -164,7 +165,7 @@ void shouldParseNodeHeaderFile() throws IOException { var headerPath = tempDir.resolve("nodes_Person_King_header.csv"); FileUtils.writeLines(headerPath.toFile(), List.of(":ID,foo:long,bar:double")); - var parsedHeader = CsvImportFileUtil.parseNodeHeader(headerPath); + var parsedHeader = CsvImportFileUtil.parseNodeHeader(headerPath, Functions.identity()); assertThat(parsedHeader.nodeLabels()).containsExactlyInAnyOrder("Person", "King"); assertThat(parsedHeader.propertyMappings()).containsExactlyInAnyOrder( diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitorTest.java index c5b98afb5f3..3a21f4f3730 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeSchemaVisitorTest.java @@ -113,6 +113,28 @@ void writesSchemaWithMixedProperties() { ); } + @Test + void shouldWriteDoubleArrayProperty() { + var nodeSchemaVisitor = new CsvNodeSchemaVisitor(tempDir); + NodeLabel labelA = NodeLabel.of("A"); + nodeSchemaVisitor.nodeLabel(labelA); + nodeSchemaVisitor.key("prop1"); + nodeSchemaVisitor.valueType(ValueType.DOUBLE_ARRAY); + nodeSchemaVisitor.defaultValue(DefaultValue.of(new double[]{42.0, 43.0})); + nodeSchemaVisitor.state(PropertyState.PERSISTENT); + nodeSchemaVisitor.endOfEntity(); + nodeSchemaVisitor.close(); + + assertCsvFiles(List.of(NODE_SCHEMA_FILE_NAME)); + assertDataContent( + NODE_SCHEMA_FILE_NAME, + List.of( + defaultHeaderColumns(), + List.of("A", "prop1", "double[]", "\"DefaultValue([42.0,43.0])\"", "PERSISTENT") + ) + ); + } + @Override protected List defaultHeaderColumns() { return NODE_SCHEMA_COLUMNS; diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index d1b080ac1ce..05d05df1e73 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -22,21 +22,21 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; -import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.graph.DoubleArrayGraphPropertyValues; import org.neo4j.gds.api.properties.graph.LongGraphPropertyValues; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfig; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfigImpl; +import org.neo4j.gds.core.loading.ArrayIdMapBuilder; import org.neo4j.gds.core.loading.Capabilities.WriteMode; +import org.neo4j.gds.core.loading.HighLimitIdMapBuilder; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; -import org.neo4j.gds.extension.GdlExtension; -import org.neo4j.gds.extension.GdlGraph; -import org.neo4j.gds.extension.Inject; import org.neo4j.gds.gdl.GdlFactory; import java.nio.file.Path; @@ -47,11 +47,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.neo4j.gds.TestSupport.assertGraphEquals; -@GdlExtension class CsvToGraphStoreImporterIntegrationTest { - @GdlGraph - private static final String GDL = + private static final String GRAPH_WITH_PROPERTIES = "CREATE" + // This triggers jackson wrapping the values in quotes " (a:A:B { averylongpropertynamegreaterthantwentyfour: 0, prop2: 42, prop3: [0.30000001192092896D, 0.20000000298023224D]})" + @@ -65,49 +63,69 @@ class CsvToGraphStoreImporterIntegrationTest { ", (c)-[:REL2 { prop3: 4, prop4: 46 }]->(d)" + ", (d)-[:REL2 { prop3: 5, prop4: 47 }]->(a)"; - @Inject - GraphStore graphStore; + private static final String GRAPH_WITH_UNDERSCORE_LABELS = + "CREATE" + + // This triggers jackson wrapping the values in quotes + " (a:A_B { averylongpropertynamegreaterthantwentyfour: 0, prop2: 42, prop3: [0.30000001192092896D, 0.20000000298023224D]})" + + ", (b:A:B { averylongpropertynamegreaterthantwentyfour: 1, prop2: 43})" + + ", (c:A:C { averylongpropertynamegreaterthantwentyfour: 2, prop2: 44, prop3: [-0.04D] })" + + ", (d:B { averylongpropertynamegreaterthantwentyfour: 3 })" + + ", (a)-[:REL1 { prop1: 0, prop2: 42 }]->(a)" + + ", (a)-[:REL1 { prop1: 1, prop2: 43 }]->(b)" + + ", (b)-[:REL1 { prop1: 2, prop2: 44 }]->(a)" + + ", (b)-[:REL2 { prop3: 3, prop4: 45 }]->(c)" + + ", (c)-[:REL2 { prop3: 4, prop4: 46 }]->(d)" + + ", (d)-[:REL2 { prop3: 5, prop4: 47 }]->(a)"; - @Inject - Graph graph; @TempDir Path graphLocation; + static Stream concurrencyLabelMappingArgs() { + return Stream.of( + Arguments.of(1, false), + Arguments.of(4, false), + Arguments.of(1, true), + Arguments.of(4, true) + ); + } + @ParameterizedTest - @ValueSource(ints = {1, 4}) - void shouldImportProperties(int concurrency) { + @MethodSource("concurrencyLabelMappingArgs") + void shouldImportProperties(int concurrency, boolean useLabelMapping) { + var graphStore = GdlFactory.of(GRAPH_WITH_PROPERTIES).build(); - GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency, useLabelMapping), graphLocation).run(); var importer = new CsvToGraphStoreImporter(concurrency, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run(); var importedGraphStore = userGraphStore.graphStore(); var importedGraph = importedGraphStore.getUnion(); - assertGraphEquals(graph, importedGraph); + assertGraphEquals(graphStore.getUnion(), importedGraph); } @ParameterizedTest - @ValueSource(ints = {1, 4}) - void shouldImportGraphStoreWithGraphProperties(int concurrency) { - addLongGraphProperty(); - addDoubleArrayGraphProperty(); - addLongNamedGraphProperty(); + @MethodSource("concurrencyLabelMappingArgs") + void shouldImportGraphStoreWithGraphProperties(int concurrency, boolean useLabelMapping) { + var graphStore = GdlFactory.of(GRAPH_WITH_PROPERTIES).build(); + + addLongGraphProperty(graphStore); + addDoubleArrayGraphProperty(graphStore); + addLongNamedGraphProperty(graphStore); - GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency, useLabelMapping), graphLocation).run(); var importer = new CsvToGraphStoreImporter(concurrency, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); - var userGraphStore = importer.run(); - var graphStore = userGraphStore.graphStore(); + var userGraphStore = importer.run().graphStore(); - assertThat(graphStore.graphPropertyKeys()).containsExactlyInAnyOrder( + assertThat(userGraphStore.graphPropertyKeys()).containsExactlyInAnyOrder( "longProp", "doubleArrayProp", "thisisaverylongnameintentionallytotriggerquoting" ); var expectedLongValues = LongStream.range(0, 10_000).toArray(); - assertThat(graphStore.graphProperty("longProp").values().longValues().toArray()) + assertThat(userGraphStore.graphProperty("longProp").values().longValues().toArray()) .containsExactlyInAnyOrder(expectedLongValues); var expectedDoubleArrayProperties = LongStream @@ -115,15 +133,30 @@ void shouldImportGraphStoreWithGraphProperties(int concurrency) { .mapToObj(i -> new double[]{(double) i, 42.0}) .collect(Collectors.toList()) .toArray(new double[0][0]); - assertThat(graphStore.graphProperty("doubleArrayProp").values().doubleArrayValues().collect(Collectors.toList())) + assertThat(userGraphStore.graphProperty("doubleArrayProp").values().doubleArrayValues().collect(Collectors.toList())) .containsExactlyInAnyOrder(expectedDoubleArrayProperties); } + @ParameterizedTest + @ValueSource(ints = {1, 4}) + void shouldImportGraphWithPropertiesAndUnderscoreLabels(int concurrency) { + var graphStore = GdlFactory.of(GRAPH_WITH_UNDERSCORE_LABELS).build(); + + GraphStoreToCsvExporter.create(graphStore, exportConfig(concurrency, true /* will not work without label mapping */), graphLocation).run(); + + var importer = new CsvToGraphStoreImporter(concurrency, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); + var userGraphStore = importer.run(); + + var importedGraphStore = userGraphStore.graphStore(); + var importedGraph = importedGraphStore.getUnion(); + assertGraphEquals(graphStore.getUnion(), importedGraph); + } + @Test void shouldImportGraphWithNoLabels() { var graphStore = GdlFactory.of("()-[]->()").build(); - GraphStoreToCsvExporter.create(graphStore, exportConfig(4), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStore, exportConfig(4, false), graphLocation).run(); var importer = new CsvToGraphStoreImporter(4, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run(); @@ -142,7 +175,7 @@ void shouldImportCapabilities(WriteMode writeMode) { .build() .build(); - GraphStoreToCsvExporter.create(graphStoreWithCapabilities, exportConfig(1), graphLocation).run(); + GraphStoreToCsvExporter.create(graphStoreWithCapabilities, exportConfig(1, false), graphLocation).run(); var importer = new CsvToGraphStoreImporter(1, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); var userGraphStore = importer.run(); @@ -154,16 +187,34 @@ void shouldImportCapabilities(WriteMode writeMode) { .isEqualTo(ImmutableStaticCapabilities.of(writeMode)); } - private GraphStoreToFileExporterConfig exportConfig(int concurrency) { + @ParameterizedTest + @ValueSource(strings = {ArrayIdMapBuilder.ID, HighLimitIdMapBuilder.ID}) + void shouldConsiderIdMapBuilderType(String idMapBuilderType) { + var graphStore = GdlFactory.builder() + .gdlGraph("()-[]->()") + .idMapBuilderType(idMapBuilderType) + .build() + .build(); + + GraphStoreToCsvExporter.create(graphStore, exportConfig(1, false), graphLocation).run(); + + var importer = new CsvToGraphStoreImporter(1, graphLocation, Neo4jProxy.testLog(), EmptyTaskRegistryFactory.INSTANCE); + var userGraphStore = importer.run(); + + assertThat(userGraphStore.graphStore().nodes().typeId()).startsWith(idMapBuilderType); + } + + private GraphStoreToFileExporterConfig exportConfig(int concurrency, boolean useLabelMapping) { return GraphStoreToFileExporterConfigImpl.builder() .exportName("my-export") .writeConcurrency(concurrency) .username("") .includeMetaData(true) + .useLabelMapping(useLabelMapping) .build(); } - private void addDoubleArrayGraphProperty() { + private void addDoubleArrayGraphProperty(GraphStore graphStore) { graphStore.addGraphProperty("doubleArrayProp", new DoubleArrayGraphPropertyValues() { @Override public Stream doubleArrayValues() { @@ -177,7 +228,7 @@ public long valueCount() { }); } - private void addLongGraphProperty() { + private void addLongGraphProperty(GraphStore graphStore) { graphStore.addGraphProperty("longProp", new LongGraphPropertyValues() { @Override public LongStream longValues() { @@ -191,7 +242,7 @@ public long valueCount() { }); } - private void addLongNamedGraphProperty() { + private void addLongNamedGraphProperty(GraphStore graphStore) { graphStore.addGraphProperty("thisisaverylongnameintentionallytotriggerquoting", new LongGraphPropertyValues() { @Override public LongStream longValues() { diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java new file mode 100644 index 00000000000..0a3e8bfcdae --- /dev/null +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/DefaultValueIOHelperTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.io.file.csv; + +import org.intellij.lang.annotations.Subst; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +class DefaultValueIOHelperTest { + + static Stream defaultValuesAndSerializedFormat() { + return Stream.of( + Arguments.of(DefaultValue.of(42.0, ValueType.DOUBLE, true), "DefaultValue(42.0)", ValueType.DOUBLE), + Arguments.of(DefaultValue.of(1337, ValueType.LONG, true), "DefaultValue(1337)", ValueType.LONG), + Arguments.of(DefaultValue.of(new Float[]{ 0.1f, 0.2f }, ValueType.FLOAT_ARRAY, true), "DefaultValue([0.1,0.2])", ValueType.FLOAT_ARRAY), + Arguments.of(DefaultValue.of(new Double[]{ 1.1, 2.2, 3.3 }, ValueType.DOUBLE_ARRAY, true), "DefaultValue([1.1,2.2,3.3])", ValueType.DOUBLE_ARRAY), + Arguments.of(DefaultValue.of(new Long[]{ 42L, 43L }, ValueType.LONG_ARRAY, true), "DefaultValue([42,43])", ValueType.LONG_ARRAY) + ); + } + + @ParameterizedTest + @MethodSource("defaultValuesAndSerializedFormat") + void shouldSerializeDefaultValues(DefaultValue defaultValue, String expected, ValueType __) { + assertThat(DefaultValueIOHelper.serialize(defaultValue)).isEqualTo(expected); + } + + @ParameterizedTest + @ValueSource(strings = { "DefaultValue(%s)", "%s" }) + void shouldDeserializedDefaultValues(@Subst("") String defaultValueTemplate) { + assertThat(DefaultValueIOHelper + .deserialize(formatWithLocale(defaultValueTemplate, "[42,43]"), ValueType.LONG_ARRAY, true) + .longArrayValue() + ).contains(42, 43); + + assertThat(DefaultValueIOHelper + .deserialize(formatWithLocale(defaultValueTemplate, "[42.0,13.37]"), ValueType.FLOAT_ARRAY, true) + .floatArrayValue() + ).contains(42.0f, 13.37f); + + assertThat(DefaultValueIOHelper + .deserialize(formatWithLocale(defaultValueTemplate, "[42.0,13.37]"), ValueType.DOUBLE_ARRAY, true) + .doubleArrayValue() + ).contains(42.0D, 13.37D); + + assertThat(DefaultValueIOHelper + .deserialize(formatWithLocale(defaultValueTemplate, "42.0"), ValueType.DOUBLE, true) + .doubleValue() + ).isEqualTo(42.0D); + + assertThat(DefaultValueIOHelper + .deserialize(formatWithLocale(defaultValueTemplate, "1337"), ValueType.LONG, true) + .longValue() + ).isEqualTo(1337L); + } +} diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java index 1d95bb5b64c..9e2ae7abbe5 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphInfoLoaderTest.java @@ -25,6 +25,8 @@ import org.junit.jupiter.api.io.TempDir; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DatabaseId; +import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.core.loading.ArrayIdMapBuilder; import java.io.IOException; import java.nio.file.Path; @@ -43,8 +45,8 @@ void shouldLoadGraphInfo(@TempDir Path exportDir) throws IOException { var databaseId = DatabaseId.from("my-database"); var graphInfoFile = exportDir.resolve(GRAPH_INFO_FILE_NAME).toFile(); var lines = List.of( - String.join(", ", "databaseName", "nodeCount", "maxOriginalId", "relTypeCounts", "inverseIndexedRelTypes"), - String.join(", ", "my-database", "19", "1337", "REL;42", "REL;REL1") + String.join(", ", "databaseName", "nodeCount", "maxOriginalId", "relTypeCounts", "inverseIndexedRelTypes","idMapBuilderType"), + String.join(", ", "my-database", "19", "1337", "REL;42", "REL;REL1", ArrayIdMapBuilder.ID) ); FileUtils.writeLines(graphInfoFile, lines); @@ -55,6 +57,8 @@ void shouldLoadGraphInfo(@TempDir Path exportDir) throws IOException { assertThat(graphInfo.databaseId()).isEqualTo(databaseId); assertThat(graphInfo.databaseId().databaseName()).isEqualTo("my-database"); + assertThat(graphInfo.idMapBuilderType()).isEqualTo(ArrayIdMapBuilder.ID); + assertThat(graphInfo.nodeCount()).isEqualTo(19L); assertThat(graphInfo.maxOriginalId()).isEqualTo(1337L); @@ -65,6 +69,21 @@ void shouldLoadGraphInfo(@TempDir Path exportDir) throws IOException { assertThat(graphInfo.inverseIndexedRelationshipTypes()).containsExactly(RelationshipType.of("REL"), RelationshipType.of("REL1")); } + @Test + void shouldHandleEmptyRelCountsAndInverseIndexRelTypes(@TempDir Path exportDir) throws IOException { + var graphInfoFile = exportDir.resolve(GRAPH_INFO_FILE_NAME).toFile(); + var lines = List.of( + String.join(", ", "databaseName", "nodeCount", "maxOriginalId", "relTypeCounts", "inverseIndexedRelTypes"), + String.join(", ", "my-database", "19", "", "") + ); + FileUtils.writeLines(graphInfoFile, lines); + + var graphInfoLoader = new GraphInfoLoader(exportDir, CSV_MAPPER); + var graphInfo = graphInfoLoader.load(); + + assertThat(graphInfo.relationshipTypeCounts()).isEmpty(); + } + /** * Test for backwards compatibility by including `databaseId` */ @@ -85,6 +104,8 @@ void shouldLoadGraphInfoWithDatabaseId(@TempDir Path exportDir) throws IOExcepti assertThat(graphInfo.databaseId()).isEqualTo(databaseId); assertThat(graphInfo.databaseId().databaseName()).isEqualTo("my-database"); + assertThat(graphInfo.idMapBuilderType()).isEqualTo(IdMap.NO_TYPE); + assertThat(graphInfo.nodeCount()).isEqualTo(19L); assertThat(graphInfo.maxOriginalId()).isEqualTo(1337L); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java index 84a8e93d644..2c82139428c 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java @@ -462,6 +462,7 @@ public long valueCount() { List.of( List.of( CsvGraphInfoVisitor.DATABASE_NAME_COLUMN_NAME, + CsvGraphInfoVisitor.ID_MAP_BUILDER_TYPE_COLUMN_NAME, CsvGraphInfoVisitor.NODE_COUNT_COLUMN_NAME, CsvGraphInfoVisitor.MAX_ORIGINAL_ID_COLUMN_NAME, CsvGraphInfoVisitor.REL_TYPE_COUNTS_COLUMN_NAME, @@ -469,6 +470,7 @@ public long valueCount() { ), List.of( graphStore.databaseId().databaseName(), + graphStore.nodes().typeId(), Long.toString(graphStore.nodeCount()), Long.toString(graphStore.nodes().highestOriginalId()), CsvMapUtil.relationshipCountsToString(Map.of(RelationshipType.of("REL2"), 6L, RelationshipType.of("REL1"), 6L)), diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/JacksonConvertersTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/JacksonConvertersTest.java deleted file mode 100644 index acd02315730..00000000000 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/JacksonConvertersTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.core.io.file.csv; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.neo4j.gds.api.DefaultValue; - -import java.util.stream.Stream; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -class JacksonConvertersTest { - - private static Stream defaultValues() { - return Stream.of( - Arguments.of(DefaultValue.forDouble().toString(), ""), - Arguments.of(DefaultValue.forLong().toString(), Long.toString(Long.MIN_VALUE)), - Arguments.of(DefaultValue.forDoubleArray().toString(), ""), - Arguments.of(DefaultValue.forLongArray().toString(), ""), - Arguments.of(DefaultValue.forFloatArray().toString(), "") - ); - } - - @ParameterizedTest - @MethodSource("defaultValues") - void shouldDeserializeDefaultValues(String defaultValue, String expected) { - var defaultValueConverter = new JacksonConverters.DefaultValueConverter(); - assertThat(defaultValueConverter.convert(defaultValue)).isEqualTo(expected); - } -} diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java index 2367508e6c9..55a8151ef49 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java @@ -144,4 +144,22 @@ void shouldLoadMixedLabels() throws IOException { .isEqualTo(new MutableNodeSchemaEntry(NodeLabel.of("B"), Map.of())); } + @Test + void shouldReadArrayDefaultValues() throws IOException { + var nodeSchemaFile = exportDir.resolve(NODE_SCHEMA_FILE_NAME).toFile(); + var lines = List.of( + String.join(", ", NODE_SCHEMA_COLUMNS), + "A, prop1, double[], \"DefaultValue([42.0,13.37])\", PERSISTENT" + ); + FileUtils.writeLines(nodeSchemaFile, lines); + + var schemaLoader = new NodeSchemaLoader(exportDir); + var nodeSchema = schemaLoader.load(); + + assertThat(nodeSchema).isNotNull(); + + assertThat(nodeSchema.availableLabels()).containsExactlyInAnyOrder(NodeLabel.of("A")); + assertThat(nodeSchema.get(NodeLabel.of("A")).properties().get("prop1").defaultValue().doubleArrayValue()) + .containsExactly(42.0, 13.37); + } } diff --git a/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java b/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java index f0014becddc..9b3fc1bf90e 100644 --- a/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java +++ b/memory-usage/src/main/java/org/neo4j/gds/mem/MemoryUsage.java @@ -364,6 +364,8 @@ private static final class VmInfoHolder { */ @SuppressForbidden(reason = "we want to use system.out here") private static boolean isVmInfoAvailable() { + macWorkaround(); + var sysOut = System.out; try { var swallowSysOut = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM, true, StandardCharsets.UTF_8); @@ -378,6 +380,21 @@ private static boolean isVmInfoAvailable() { } } + /** + * JOL currently kills the JVM on Mac OS X when trying to attach to the Hotspot SA. + * This happens with several JVMs (Java.net, Azul) and on both platforms (x86, ARM). + *

+ * We can work around this by skipping the Hotspot SA attach. + * See OpenJDK issue + *

+ */ + @Deprecated(forRemoval = true) + private static void macWorkaround() { + if (System.getProperty("os.name").contains("Mac")) { + System.setProperty("jol.skipHotspotSAAttach", "true"); + } + } + private static final class NullOutputStream extends OutputStream { static final OutputStream NULL_OUTPUT_STREAM = new NullOutputStream(); diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java index 769b63caf6b..d5982ac9513 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java @@ -30,7 +30,7 @@ import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; @@ -170,7 +170,7 @@ private static RelationshipsBuilder newRelationshipsBuilder( .map(key -> List.of(GraphFactory.PropertyConfig.of(key, Aggregation.SINGLE, DefaultValue.forDouble()))) .orElse(List.of())) .concurrency(1) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); } diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java index afd111f4e05..bc893371c5a 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java @@ -32,6 +32,8 @@ public interface ModelCatalog { void registerListener(ModelCatalogListener listener); + void unregisterListener(ModelCatalogListener listener); + void set(Model model); Model get( @@ -78,6 +80,11 @@ public void registerListener(ModelCatalogListener listener) { } + @Override + public void unregisterListener(ModelCatalogListener listener) { + + } + @Override public void set(Model model) { diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java index 33451b2d659..0301c3aac48 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java @@ -29,10 +29,13 @@ import org.neo4j.graphdb.NotFoundException; import org.neo4j.graphdb.Result; import org.neo4j.graphdb.Transaction; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; import org.neo4j.internal.kernel.api.security.LoginContext; import org.neo4j.io.layout.DatabaseLayout; import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.CallableUserFunction; import org.neo4j.kernel.api.procedure.GlobalProcedures; import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.impl.coreapi.InternalTransaction; @@ -40,10 +43,14 @@ import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.storageengine.api.StorageEngineFactory; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import java.util.function.Function; + public final class GraphDatabaseApiProxy { public static Neo4jVersion neo4jVersion() { @@ -82,7 +89,7 @@ public static void registerProcedures( ) throws KernelException { GlobalProcedures procedures = resolveDependency(db, GlobalProcedures.class); for (Class clazz : procedureClasses) { - procedures.registerProcedure(clazz, overrideCurrentImplementation); + registerProcedures(procedures, clazz, overrideCurrentImplementation); } } @@ -105,7 +112,24 @@ public static void registerAggregationFunctions(GraphDatabaseService db, Class procedureClass, + boolean overrideCurrentImplementation + ) throws KernelException { + var globalProceduresClass = globalProcedures.getClass(); + + var legacySignatureMethod = getMethod(globalProceduresClass, "registerProcedure", Class.class, boolean.class); + var newSignatureMethod = getMethod(globalProceduresClass, "registerProcedure", Class.class); + if (legacySignatureMethod.isPresent()) { + invokeMethod(globalProcedures, legacySignatureMethod.get(), procedureClass, overrideCurrentImplementation); + } else if (newSignatureMethod.isPresent()){ + invokeMethod(globalProcedures, newSignatureMethod.get(), procedureClass); + } else { + throw new RuntimeException("Could not find registerProcedure method"); + } + } + + private static void register(GlobalProcedures globalProcedures, Class clazz, Object obj) throws KernelException { + var globalProceduresClass = globalProcedures.getClass(); + + var legacySignatureMethod = getMethod(globalProceduresClass, "register", clazz, boolean.class); + var newSignatureMethod = getMethod(globalProceduresClass, "register", clazz); + if (legacySignatureMethod.isPresent()) { + invokeMethod(globalProcedures, legacySignatureMethod.get(), obj, true); + } else if (newSignatureMethod.isPresent()){ + invokeMethod(globalProcedures, newSignatureMethod.get(), obj); + } else { + throw new RuntimeException("Could not find registerProcedure method"); + } + } + + private static Optional getMethod(Class clazz, String name, Class... parameterTypes) { + try { + return Optional.of(clazz.getMethod(name, parameterTypes)); + } catch (NoSuchMethodException e) { + return Optional.empty(); + } + } + + private static void invokeMethod(Object obj, Method method, Object... arguments) + throws KernelException { + try { + try { + method.invoke(obj, arguments); + } catch (InvocationTargetException e) { + if (e.getTargetException().getClass() != ProcedureException.class) { + throw e.getTargetException(); + } + } + } catch (KernelException | RuntimeException e) { + throw e; + } catch (Throwable e) { + throw new RuntimeException(e); + } + } } diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 0cb402662ec..69a6f098a1a 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -36,7 +36,12 @@ public enum Neo4jVersion { V_5_6, V_5_7, V_5_8, - V_Dev; + V_5_9, + V_5_10, + V_5_11, + V_5_12, + V_5_13, + V_RC; @Override public String toString() { @@ -59,16 +64,26 @@ public String toString() { return "5.7"; case V_5_8: return "5.8"; - case V_Dev: - return "dev"; + case V_5_9: + return "5.9"; + case V_5_10: + return "5.10"; + case V_5_11: + return "5.11"; + case V_5_12: + return "5.12"; + case V_5_13: + return "5.13"; + case V_RC: + return "rc"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } } public MajorMinorVersion semanticVersion() { - if (this == V_Dev) { - return ImmutableMajorMinorVersion.of(5, 9); + if (this == V_RC) { + return ImmutableMajorMinorVersion.of(5, 13); } String version = toString(); @@ -147,8 +162,18 @@ static Neo4jVersion parse(String version) { return Neo4jVersion.V_5_7; } else if (minorVersion == 8) { return Neo4jVersion.V_5_8; - } else if (minorVersion > 8) { - return Neo4jVersion.V_Dev; + } else if (minorVersion == 9) { + return Neo4jVersion.V_5_9; + } else if (minorVersion == 10) { + return Neo4jVersion.V_5_10; + } else if (minorVersion == 11) { + return Neo4jVersion.V_5_11; + } else if (minorVersion == 12) { + return Neo4jVersion.V_5_12; + } else if (minorVersion == 13) { + return Neo4jVersion.V_5_13; + } else if (minorVersion == 14) { + return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 295f61e7190..bdf5f742b30 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -49,7 +49,12 @@ class Neo4jVersionTest { "5.6.0, V_5_6", "5.7.0, V_5_7", "5.8.0, V_5_8", - "5.9.0, V_Dev", + "5.9.0, V_5_9", + "5.10.0, V_5_10", + "5.11.0, V_5_11", + "5.12.0, V_5_12", + "5.13.0, V_5_13", + "5.14.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); @@ -90,6 +95,11 @@ void shouldNotRespectVersionOverride() { "5.6.0, 5, 6", "5.7.0, 5, 7", "5.8.0, 5, 8", + "5.9.0, 5, 9", + "5.10.0, 5, 10", + "5.11.0, 5, 11", + "5.12.0, 5, 12", + "5.13.0, 5, 13", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); diff --git a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java index cf95b92bbbf..958556e1efa 100644 --- a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java +++ b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java @@ -28,10 +28,11 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.List; +import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; @@ -41,11 +42,11 @@ public final class OpenModelCatalog implements ModelCatalog { private final Map userCatalogs; - private final List listeners; + private final Set listeners; public OpenModelCatalog() { this.userCatalogs = new ConcurrentHashMap<>(); - this.listeners = new ArrayList<>(); + this.listeners = new HashSet<>(); } @Override @@ -53,6 +54,11 @@ public void registerListener(ModelCatalogListener listener) { listeners.add(listener); } + @Override + public void unregisterListener(ModelCatalogListener listener) { + listeners.remove(listener); + } + @Override public void set(Model model) { userCatalogs.compute(model.creator(), (user, userCatalog) -> { diff --git a/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java b/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java index bcbe9df3dc7..639e49c65f4 100644 --- a/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java +++ b/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java @@ -182,10 +182,8 @@ class OpenGdsProcedureSmokeTest extends BaseProcTest { "gds.alpha.scc.write", "gds.alpha.scc.stream", - "gds.alpha.scaleProperties.mutate", "gds.scaleProperties.mutate", "gds.scaleProperties.mutate.estimate", - "gds.alpha.scaleProperties.stream", "gds.scaleProperties.stream", "gds.scaleProperties.stream.estimate", "gds.scaleProperties.stats", @@ -562,7 +560,7 @@ void countShouldMatch() { ); // If you find yourself updating this count, please also update the count in SmokeTest.kt - int expectedCount = 407; + int expectedCount = 405; assertEquals( expectedCount, returnedRows, diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStep.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStep.java index cf7e7c93447..1abd73bb46b 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStep.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStep.java @@ -43,7 +43,7 @@ import java.util.Objects; import java.util.stream.Collectors; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.ml.pipeline.NodePropertyStepContextConfig.CONTEXT_NODE_LABELS; import static org.neo4j.gds.ml.pipeline.NodePropertyStepContextConfig.CONTEXT_RELATIONSHIP_TYPES; diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java index 1343703a997..ea53bbefe36 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/NodePropertyStepFactory.java @@ -100,6 +100,8 @@ public static GdsCallableFinder.GdsCallableDefinition getGdsCallableDefinition(S var gdsCallableDefinition = GdsCallableFinder .findByName(normalizedName) + // If this is thrown in a production setting, it may be that you just need to add `annotationProcessor project(':procedure-collector')` + // to the `build.gradle` file of the package containing your proc .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( "Could not find a procedure called %s", normalizedName diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/Pipeline.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/Pipeline.java index 9387a5f07e8..4a79a398f35 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/Pipeline.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/Pipeline.java @@ -29,7 +29,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public interface Pipeline extends ToMapConvertible { diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/TrainingPipeline.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/TrainingPipeline.java index 078490a1c4b..5f1e77fb4d4 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/TrainingPipeline.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/TrainingPipeline.java @@ -32,7 +32,7 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public abstract class TrainingPipeline implements Pipeline { diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/ExecutableNodePropertyStepTestUtil.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/ExecutableNodePropertyStepTestUtil.java index 2e5f879407c..258116ab46f 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/ExecutableNodePropertyStepTestUtil.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/ExecutableNodePropertyStepTestUtil.java @@ -38,7 +38,7 @@ import java.util.Optional; import java.util.UUID; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; public class ExecutableNodePropertyStepTestUtil { diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java index 9b3a7a99cdf..e897aeb0b63 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java @@ -27,11 +27,11 @@ import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; import org.neo4j.gds.api.TerminationMonitor; import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.model.OpenModelCatalog; @@ -182,7 +182,7 @@ void overridesTheSplitConfig() { void deriveRelationshipWeightProperty() { var executionContext = ImmutableExecutionContext.builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .username("") .terminationMonitor(TerminationMonitor.EMPTY) .closeableResourceRegistry(CloseableResourceRegistry.EMPTY) @@ -226,7 +226,7 @@ void deriveRelationshipWeightPropertyFromTrainedModel() { var executionContext = ImmutableExecutionContext.builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .username("") .modelCatalog(modelCatalog) .terminationMonitor(TerminationMonitor.EMPTY) @@ -270,7 +270,7 @@ void notDerivePropertyFromUnweightedTrainedModel() { var executionContext = ImmutableExecutionContext.builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .username("") .modelCatalog(modelCatalog) .terminationMonitor(TerminationMonitor.EMPTY) diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrainPipelineExecutorTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrainPipelineExecutorTest.java index 57a84c9f9f3..2a6f2dc266d 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrainPipelineExecutorTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrainPipelineExecutorTest.java @@ -82,7 +82,7 @@ import static org.neo4j.gds.TestSupport.assertMemoryRange; import static org.neo4j.gds.assertj.Extractors.keepingFixedNumberOfDecimals; import static org.neo4j.gds.assertj.Extractors.removingThreadId; -import static org.neo4j.gds.config.MutatePropertyConfig.MUTATE_PROPERTY_KEY; +import static org.neo4j.gds.config.MutateNodePropertyConfig.MUTATE_PROPERTY_KEY; import static org.neo4j.gds.ml.pipeline.ExecutableNodePropertyStepTestUtil.TestNodePropertyStepWithFixedEstimation; import static org.neo4j.gds.ml.pipeline.PipelineExecutor.DatasetSplits.FEATURE_INPUT; import static org.neo4j.gds.ml.pipeline.PipelineExecutor.DatasetSplits.TEST; diff --git a/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java b/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java index 1b767603d90..35e73d4348c 100644 --- a/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java +++ b/pregel-proc-generator/src/main/java/org/neo4j/gds/pregel/generator/AlgorithmGenerator.java @@ -29,7 +29,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelResult; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -88,9 +88,9 @@ MethodSpec constructor() { .addStatement("super(progressTracker)") .addStatement("var computation = new $T()", typeNames.computation()) .addStatement( - "this.pregelJob = $T.create(graph, configuration, computation, $T.DEFAULT, progressTracker)", + "this.pregelJob = $T.create(graph, configuration, computation, $T.INSTANCE, progressTracker)", Pregel.class, - Pools.class + DefaultPool.class ).build(); } diff --git a/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java b/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java index 381a9936b03..d04e6312928 100644 --- a/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java +++ b/pregel-proc-generator/src/test/java/org/neo4j/gds/pregel/generator/AlgorithmGeneratorTest.java @@ -61,7 +61,7 @@ void shouldGenerateConstructor() { " org.neo4j.gds.core.utils.progress.tasks.ProgressTracker progressTracker) {" + NL + " super(progressTracker);" + NL + " var computation = new a.b.C();" + NL + - " this.pregelJob = org.neo4j.gds.beta.pregel.Pregel.create(graph, configuration, computation, org.neo4j.gds.core.concurrency.Pools.DEFAULT, progressTracker);" + NL + + " this.pregelJob = org.neo4j.gds.beta.pregel.Pregel.create(graph, configuration, computation, org.neo4j.gds.core.concurrency.DefaultPool.INSTANCE, progressTracker);" + NL + "}" + NL ); } diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java index a42fa3e1a19..d682a26ab71 100644 --- a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java @@ -25,7 +25,7 @@ import org.neo4j.gds.beta.pregel.Pregel; import org.neo4j.gds.beta.pregel.PregelProcedureConfig; import org.neo4j.gds.beta.pregel.PregelResult; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -37,7 +37,7 @@ public final class BidirectionalComputationAlgorithm extends Algorithm { ProgressTracker progressTracker) { super(progressTracker); var computation = new Computation(); - this.pregelJob = Pregel.create(graph, configuration, computation, Pools.DEFAULT, progressTracker); + this.pregelJob = Pregel.create(graph, configuration, computation, DefaultPool.INSTANCE, progressTracker); } @Override diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java index 56ce22a56cd..8c8a788f7b2 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/Pregel.java @@ -22,7 +22,7 @@ import org.immutables.value.Value; import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.context.MasterComputeContext; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -159,7 +159,7 @@ private Pregel( .messenger(messenger) .voteBits(HugeAtomicBitSet.create(graph.nodeCount())) .executorService(config.useForkJoin() - ? Pools.createForkJoinPool(config.concurrency()) + ? ExecutorServiceUtil.createForkJoinPool(config.concurrency()) : executor) .progressTracker(progressTracker) .build(); diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java index 0f9d0fa81d0..2a2c5687584 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PregelProcedureConfig.java @@ -20,19 +20,24 @@ package org.neo4j.gds.beta.pregel; import org.immutables.value.Value; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.config.WritePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; +import java.util.Collection; + @ValueClass @Configuration @SuppressWarnings("immutables:subtype") public interface PregelProcedureConfig extends PregelConfig, WritePropertyConfig, - MutatePropertyConfig { + MutateNodePropertyConfig { @Value.Default default String writeProperty() { @@ -44,6 +49,29 @@ default String mutateProperty() { return ""; } + @Override + @Configuration.GraphStoreValidationCheck + @Value.Default + default void validateGraphIsSuitableForWrite( + GraphStore graphStore, + @SuppressWarnings("unused") Collection selectedLabels, + @SuppressWarnings("unused") Collection selectedRelationshipTypes + ) { + // VN/IP: HACK! + // Since we are using the same configuration for all the modes (`stream`, `write` and `mutate`) we check if the + // graph is writable in all modes. + // We only want to raise the error if the user actually doing writes, + // which we assume is when there is a `writeProperty` present + if (writeProperty().isBlank()) { + return; + } + + if (!graphStore.capabilities().canWriteToDatabase() && !graphStore.capabilities().canWriteToRemoteDatabase()) { + throw new IllegalArgumentException("The provided graph does not support `write` execution mode."); + } + } + + static PregelProcedureConfig of(CypherMapWrapper userInput) { return new PregelProcedureConfigImpl(userInput); } diff --git a/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java b/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java index 8dc1deac7ca..d65e7879a08 100644 --- a/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java +++ b/pregel/src/test/java/org/neo4j/gds/beta/pregel/PregelTest.java @@ -44,7 +44,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.ImmutableGraphDimensions; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; @@ -109,7 +109,7 @@ void sendsMessages( graph, config, computation, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -127,7 +127,7 @@ void stopsEarlyWhenTransactionHasBeenTerminated() { graph, config, new TestPregelComputation(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); pregelJob.setTerminationFlag(terminationFlag); @@ -163,7 +163,7 @@ void logProgress(Partitioning partitioning) { graph, config, computation, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ).run(); @@ -225,7 +225,7 @@ void cleanupProgressLogging() { graph, config, computation, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); @@ -280,7 +280,7 @@ private HugeDoubleArray run(Graph graph, PregelConfig config, PregelComputation< graph, config, computation, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -300,7 +300,7 @@ void sendMessageToSpecificTarget(Partitioning partitioning) { graph, config, new TestSendTo(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -325,7 +325,7 @@ void compositeNodeValueTest(Partitioning partitioning) { graph, config, new CompositeTestComputation(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -363,7 +363,7 @@ void testMasterComputeStep(Partitioning partitioning) { graph, ImmutablePregelConfig.builder().maxIterations(4).partitioning(partitioning).build(), new TestMasterCompute(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -378,7 +378,7 @@ void testMasterComputeStepWithConvergence(Partitioning partitioning) { graph, ImmutablePregelConfig.builder().maxIterations(4).partitioning(partitioning).build(), new TestMasterCompute(2), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -515,7 +515,7 @@ public boolean masterCompute(MasterComputeContext context) { return true; } }, - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ).run(); @@ -603,7 +603,7 @@ void preventIllegalConcurrencyConfiguration(Partitioning partitioning) { graph, config, new TestSendTo(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER )); } @@ -626,7 +626,7 @@ void messagesInInitialSuperStepShouldBeEmpty(Partitioning partitioning, boolean graph, config, new TestEmptyMessageInInitialSuperstep(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -825,7 +825,7 @@ void throwIfBidirectionalWithoutInverseIndex() { graph, ImmutablePregelConfig.builder().maxIterations(4).build(), new Bidirectional(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java index c73c97037d9..d3144ba9095 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphProjectProc.java @@ -30,7 +30,7 @@ import org.neo4j.gds.config.GraphProjectFromGraphConfig; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.mem.MemoryTree; @@ -232,7 +232,7 @@ private GraphProjectSubgraphResult projectGraphFromGraphStore( var graphStore = GraphStoreFilter.filter( fromGraphStore, config, - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java index 251a0c8310f..6ed93cb4182 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphSampleProc.java @@ -47,6 +47,7 @@ public class GraphSampleProc extends CatalogProc { private static final String CNARW_DESCRIPTION = "Constructs a random subgraph based on common neighbour aware random walks"; @Internal + @Deprecated(forRemoval = true) @Procedure(name = "gds.alpha.graph.sample.rwr", mode = READ, deprecatedBy = "gds.graph.sample.rwr") @Description(RWR_DESCRIPTION) public Stream sampleRandomWalkWithRestartsAlpha( diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java index 336f7382504..59aa4375451 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStoreExportProc.java @@ -75,10 +75,7 @@ public Stream database( validateGraphStore(graphStore, exportConfig); var progressTracker = new TaskProgressTracker( - ProgressTrackerExecutionMonitor.progressTask( - graphStore.nodeCount(), - graphStore.relationshipCount() - ), + ProgressTrackerExecutionMonitor.progressTask(), executionContext().log(), exportConfig.writeConcurrency(), exportConfig.jobId(), @@ -95,20 +92,28 @@ public Stream database( progressTracker ); - var start = System.nanoTime(); - var exportedProperties = exporter.run(); - var end = System.nanoTime(); - - return new DatabaseExportResult( - graphName, - exportConfig.dbName(), - graphStore.nodeCount(), - graphStore.relationshipCount(), - graphStore.relationshipTypes().size(), - exportedProperties.nodePropertyCount(), - exportedProperties.relationshipPropertyCount(), - java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(end - start) - ); + try { + var start = System.nanoTime(); + var exportedProperties = exporter.run(); + var end = System.nanoTime(); + + return new DatabaseExportResult( + graphName, + exportConfig.dbName(), + graphStore.nodeCount(), + graphStore.relationshipCount(), + graphStore.relationshipTypes().size(), + exportedProperties.nodePropertyCount(), + exportedProperties.relationshipPropertyCount(), + java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(end - start) + ); + } catch (Exception e) { + // Ideally we should not have this logic on the proc level + // the progress tracker is instantiated in this proc + // so we need to make sure it closes as well + progressTracker.endSubTaskWithFailure(); + throw e; + } } ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProc.java index d8f3ebf3318..0cdf5ea7629 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProc.java @@ -33,9 +33,11 @@ import org.neo4j.procedure.Procedure; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -93,7 +95,7 @@ public Stream streamNodeProperty( configuration, List.of(nodeProperty), nodeLabels, - (nodeId, propertyName, propertyValue) -> new PropertyResult(nodeId, propertyValue) + (nodeId, propertyName, propertyValue, nodeLabelList) -> new PropertyResult(nodeId, propertyValue, nodeLabelList) ); } @@ -111,7 +113,7 @@ public Stream streamProperty( configuration, List.of(nodeProperty), nodeLabels, - (nodeId, propertyName, propertyValue) -> new PropertyResult(nodeId, propertyValue), + (nodeId, propertyName, propertyValue, nodeLabelList) -> new PropertyResult(nodeId, propertyValue, nodeLabelList), Optional.of(deprecationWarning) ); } @@ -173,6 +175,10 @@ private Stream streamNodeProperties( deprecationWarning.ifPresent(taskProgressTracker::logWarning); + Function> nodeLabelsFn = config.listNodeLabels() + ? nodeId -> subGraph.nodeLabels(nodeId).stream().map(NodeLabel::name).collect(Collectors.toList()) + : nodeId -> Collections.emptyList(); + var resultStream = LongStream .range(0, subGraph.nodeCount()) .boxed() @@ -184,7 +190,8 @@ private Stream streamNodeProperties( return producer.produce( originalId, usesPropertyNameColumn ? propertyKeyAndValues.getKey() : null, - propertyKeyAndValues.getValue().getObject(nodeId) + propertyKeyAndValues.getValue().getObject(nodeId), + nodeLabelsFn.apply(nodeId) ); }); }) @@ -201,11 +208,13 @@ public static class PropertiesResult { public final long nodeId; public final String nodeProperty; public final Object propertyValue; + public final List nodeLabels; - PropertiesResult(long nodeId, String nodeProperty, Object propertyValue) { + PropertiesResult(long nodeId, String nodeProperty, Object propertyValue, List nodeLabels) { this.nodeId = nodeId; this.nodeProperty = nodeProperty; this.propertyValue = propertyValue; + this.nodeLabels = nodeLabels; } } @@ -213,15 +222,17 @@ public static class PropertiesResult { public static class PropertyResult { public final long nodeId; public final Object propertyValue; + public final List nodeLabels; - PropertyResult(long nodeId, Object propertyValue) { + PropertyResult(long nodeId, Object propertyValue, List nodeLabels) { this.nodeId = nodeId; this.propertyValue = propertyValue; + this.nodeLabels = nodeLabels; } } interface ResultProducer { - R produce(long nodeId, String propertyName, Object propertyValue); + R produce(long nodeId, String propertyName, Object propertyValue, List nodeLabels); } } diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java index 7cf025dab39..d05279ca8a0 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java @@ -21,7 +21,7 @@ import org.neo4j.gds.beta.filter.NodesFilter; import org.neo4j.gds.config.WriteLabelConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -66,7 +66,7 @@ public Stream write( nodeFilter, procedureConfig.concurrency(), Map.of(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); @@ -74,7 +74,7 @@ public Stream write( .withIdMap(filteredNodes.idMap()) .withTerminationFlag(TerminationFlag.wrap(executionContext().terminationMonitor())) .withArrowConnectionInfo(procedureConfig.arrowConnectionInfo(), graphStore.databaseId().databaseName()) - .parallel(Pools.DEFAULT, procedureConfig.concurrency()) + .parallel(DefaultPool.INSTANCE, procedureConfig.concurrency()) .build(); runWithExceptionLogging( diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java index 7e6de0142c8..f660e7fa6d4 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeLabelMutator.java @@ -22,7 +22,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.beta.filter.NodesFilter; import org.neo4j.gds.config.MutateLabelConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.loading.GraphStoreWithConfig; import org.neo4j.gds.core.loading.ImmutableCatalogRequest; @@ -52,7 +52,7 @@ static Stream mutateNodeLabel(String graphName, String nodeLa nodeFilter, procedureConfig.concurrency(), Map.of(), - Pools.DEFAULT, + DefaultPool.INSTANCE, ProgressTracker.NULL_TRACKER ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java index 4f23f0e456e..7a338f2c15b 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodePropertiesWriter.java @@ -27,7 +27,7 @@ import org.neo4j.gds.config.GraphWriteNodePropertiesConfig; import org.neo4j.gds.core.CypherMapAccess; import org.neo4j.gds.core.CypherMapWrapper; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.GraphStoreWithConfig; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.TerminationFlag; @@ -147,7 +147,7 @@ private static long writeNodeProperties( var exporter = nodePropertyExporterBuilder .withIdMap(subGraph) .withTerminationFlag(TerminationFlag.wrap(executionContext.terminationMonitor())) - .parallel(Pools.DEFAULT, config.writeConcurrency()) + .parallel(DefaultPool.INSTANCE, config.writeConcurrency()) .withProgressTracker(progressTracker) .withArrowConnectionInfo(config.arrowConnectionInfo(), graphStore.databaseId().databaseName()) .build(); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/UserInputAsStringOrListOfString.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/UserInputAsStringOrListOfString.java index 71ae76da9ff..4f02d87dd70 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/UserInputAsStringOrListOfString.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/UserInputAsStringOrListOfString.java @@ -36,7 +36,7 @@ private UserInputAsStringOrListOfString() {} public static List parse(Object userInput, String configurationKey) { if (userInput instanceof Iterable) { var result = new ArrayList(); - for (Object item : (Iterable) userInput) { + for (Object item : (Iterable) userInput) { result.add(parseOne(item, configurationKey)); } return result; diff --git a/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java b/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java index 8d61253784b..648af2657eb 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/config/GraphExportNodePropertiesConfig.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.config; +import org.immutables.value.Value; import org.neo4j.gds.ElementProjection; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.Configuration; @@ -34,12 +35,20 @@ public interface GraphExportNodePropertiesConfig extends GraphNodePropertiesConfig { + boolean LIST_NODE_LABELS_DEFAULT_VALUE = false; + @Configuration.Parameter @Configuration.ConvertWith(method = "org.neo4j.gds.config.GraphExportNodePropertiesConfig#parseNodeProperties") List nodeProperties(); + @Value.Default + @Value.Parameter(false) + @Configuration.Key("listNodeLabels") + default boolean listNodeLabels() { + return LIST_NODE_LABELS_DEFAULT_VALUE; + } + static List parseNodeProperties(Object userInput) { - return UserInputAsStringOrListOfString.parse(userInput, "nodeProperties"); } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListOperatorTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListOperatorTest.java index f2cb58cee59..8f26877881d 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListOperatorTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListOperatorTest.java @@ -35,7 +35,6 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.TerminationMonitor; @@ -438,7 +437,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder(String usernam return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(fieldName -> returnFields.contains(fieldName)) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMemoryUsageTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMemoryUsageTest.java index 4c5e8ec5fe4..8181e9ec228 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMemoryUsageTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMemoryUsageTest.java @@ -26,7 +26,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; @@ -127,7 +126,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder() { return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcEstimateTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcEstimateTest.java index 22d215e8056..88b03872e3b 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcEstimateTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcEstimateTest.java @@ -108,8 +108,8 @@ void virtualEstimateHeapPercentage(SoftAssertions softly) { runQueryWithRowConsumer(query, map("relProjection", relProjection), row -> { - softly.assertThat(row.getNumber("bytesMin").longValue()).isEqualTo(68686368); - softly.assertThat(row.getNumber("bytesMax").longValue()).isEqualTo(68686368); + softly.assertThat(row.getNumber("bytesMin").longValue()).isEqualTo(68686120); + softly.assertThat(row.getNumber("bytesMax").longValue()).isEqualTo(68686120); softly.assertThat(row.getNumber("heapPercentageMin").doubleValue()).isEqualTo(expectedPercentage); softly.assertThat(row.getNumber("heapPercentageMax").doubleValue()).isEqualTo(expectedPercentage); } @@ -207,8 +207,8 @@ void computeMemoryEstimationForVirtualGraphWithLargeValues(SoftAssertions softly runQueryWithRowConsumer( query, row -> { - softly.assertThat(row.getNumber("bytesMin").longValue()).isEqualTo(344128414784L); - softly.assertThat(row.getNumber("bytesMax").longValue()).isEqualTo(384930604096L); + softly.assertThat(row.getNumber("bytesMin").longValue()).isEqualTo(344128414632L); + softly.assertThat(row.getNumber("bytesMax").longValue()).isEqualTo(384930603944L); softly.assertThat(row.getNumber("nodeCount").longValue()).isEqualTo(5_000_000_000L); softly.assertThat(row.getNumber("relationshipCount").longValue()).isEqualTo(20_000_000_000L); } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java index 040467e68e9..22feadcb189 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java @@ -47,9 +47,9 @@ import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.Aggregation; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistry; @@ -895,7 +895,7 @@ void loadGraphWithSaturatedThreadPool() { // block all available threads for (int i = 0; i < ConcurrencyConfig.DEFAULT_CONCURRENCY; i++) { futures.add( - Pools.DEFAULT.submit(() -> LockSupport.parkNanos(Duration.ofSeconds(1).toNanos())) + DefaultPool.INSTANCE.submit(() -> LockSupport.parkNanos(Duration.ofSeconds(1).toNanos())) ); } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java index d1426b048b6..3aa9524a977 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java @@ -44,6 +44,8 @@ import org.neo4j.test.TestDatabaseManagementServiceBuilder; import org.neo4j.test.extension.ExtensionCallback; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -129,32 +131,32 @@ void tearDown() { "CALL gds.graph.nodeProperties.stream(" + " '%s', " + " ['newNodeProp1', 'newNodeProp2']" + - ") YIELD nodeId, nodeProperty, propertyValue " + - "RETURN nodeId AS id, nodeProperty, propertyValue", + ") YIELD nodeId, nodeProperty, propertyValue, nodeLabels " + + "RETURN nodeId AS id, nodeProperty, propertyValue, nodeLabels", // explicit PROJECT_ALL "CALL gds.graph.nodeProperties.stream(" + " '%s', " + " ['newNodeProp1', 'newNodeProp2'], " + " ['*']" + - ") YIELD nodeId, nodeProperty, propertyValue " + - "RETURN nodeId AS id, nodeProperty, propertyValue" + ") YIELD nodeId, nodeProperty, propertyValue, nodeLabels " + + "RETURN nodeId AS id, nodeProperty, propertyValue, nodeLabels" }) void streamLoadedNodeProperties(String graphWriteQueryTemplate) { String graphWriteQuery = formatWithLocale(graphWriteQueryTemplate, TEST_GRAPH_SAME_PROPERTIES); assertCypherResult(graphWriteQuery, asList( - map("id", idFunction.of("a"), "nodeProperty", "newNodeProp1", "propertyValue", 0D), - map("id", idFunction.of("a"), "nodeProperty", "newNodeProp2", "propertyValue", 42L), - map("id", idFunction.of("b"), "nodeProperty", "newNodeProp1", "propertyValue", 1D), - map("id", idFunction.of("b"), "nodeProperty", "newNodeProp2", "propertyValue", 43L), - map("id", idFunction.of("c"), "nodeProperty", "newNodeProp1", "propertyValue", 2D), - map("id", idFunction.of("c"), "nodeProperty", "newNodeProp2", "propertyValue", 44L), - map("id", idFunction.of("d"), "nodeProperty", "newNodeProp1", "propertyValue", 3D), - map("id", idFunction.of("d"), "nodeProperty", "newNodeProp2", "propertyValue", 45L), - map("id", idFunction.of("e"), "nodeProperty", "newNodeProp1", "propertyValue", 4D), - map("id", idFunction.of("e"), "nodeProperty", "newNodeProp2", "propertyValue", 46L), - map("id", idFunction.of("f"), "nodeProperty", "newNodeProp1", "propertyValue", 5D), - map("id", idFunction.of("f"), "nodeProperty", "newNodeProp2", "propertyValue", 47L) + map("id", idFunction.of("a"), "nodeProperty", "newNodeProp1", "propertyValue", 0D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("a"), "nodeProperty", "newNodeProp2", "propertyValue", 42L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("b"), "nodeProperty", "newNodeProp1", "propertyValue", 1D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("b"), "nodeProperty", "newNodeProp2", "propertyValue", 43L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("c"), "nodeProperty", "newNodeProp1", "propertyValue", 2D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("c"), "nodeProperty", "newNodeProp2", "propertyValue", 44L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("d"), "nodeProperty", "newNodeProp1", "propertyValue", 3D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("d"), "nodeProperty", "newNodeProp2", "propertyValue", 45L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("e"), "nodeProperty", "newNodeProp1", "propertyValue", 4D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("e"), "nodeProperty", "newNodeProp2", "propertyValue", 46L, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("f"), "nodeProperty", "newNodeProp1", "propertyValue", 5D, "nodeLabels", Collections.emptyList()), + map("id", idFunction.of("f"), "nodeProperty", "newNodeProp2", "propertyValue", 47L, "nodeLabels", Collections.emptyList()) )); } @@ -240,6 +242,23 @@ void streamMutatedNodeProperties() { )); } + @Test + void shouldListLabelsIfConfigIsSet() { + assertCypherResult( + "CALL gds.graph.nodeProperties.stream($graph, 'newNodeProp1', ['*'], {listNodeLabels: true}) " + + " YIELD nodeId, nodeProperty, propertyValue, nodeLabels " + + " RETURN nodeId AS id, nodeProperty, propertyValue, nodeLabels", + Map.of("graph", TEST_GRAPH_DIFFERENT_PROPERTIES), + asList( + Map.of("id", idFunction.of("a"), "nodeProperty", "newNodeProp1", "propertyValue", 0D, "nodeLabels", List.of("A")), + Map.of("id", idFunction.of("b"), "nodeProperty", "newNodeProp1", "propertyValue", 1D, "nodeLabels", List.of("A")), + Map.of("id", idFunction.of("c"), "nodeProperty", "newNodeProp1", "propertyValue", 2D, "nodeLabels", List.of("A")), + Map.of("id", idFunction.of("d"), "nodeProperty", "newNodeProp1", "propertyValue", 3D, "nodeLabels", List.of("B")), + Map.of("id", idFunction.of("e"), "nodeProperty", "newNodeProp1", "propertyValue", 4D, "nodeLabels", List.of("B")), + Map.of("id", idFunction.of("f"), "nodeProperty", "newNodeProp1", "propertyValue", 5D, "nodeLabels", List.of("B")) + )); + } + @Test void shouldFailOnNonExistingNodeProperties() { assertError( diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeLabelMutatorTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeLabelMutatorTest.java index 1839ca86465..017e817b977 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeLabelMutatorTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeLabelMutatorTest.java @@ -29,7 +29,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; @@ -298,7 +297,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder() { return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodePropertiesWriterTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodePropertiesWriterTest.java index fe4e366675f..5bffb6b0694 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodePropertiesWriterTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodePropertiesWriterTest.java @@ -32,7 +32,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; @@ -310,7 +309,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder() { return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/SamplerOperatorTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/SamplerOperatorTest.java index 1435d6b0ce6..97151c0ba36 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/SamplerOperatorTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/SamplerOperatorTest.java @@ -27,7 +27,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; import org.neo4j.gds.api.ProcedureReturnColumns; @@ -233,7 +232,7 @@ private ImmutableExecutionContext.Builder executionContextBuilder() { return ImmutableExecutionContext .builder() .databaseId(graphStore.databaseId()) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/centrality/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityStreamSpec.java b/proc/centrality/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityStreamSpec.java index 3fadaf2c7df..d9385388dfd 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityStreamSpec.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/closeness/ClosenessCentralityStreamSpec.java @@ -21,7 +21,6 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.common.CentralityStreamResult; -import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -31,6 +30,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.closeness.ClosenessCentrality.CLOSENESS_DESCRIPTION; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @@ -54,26 +54,22 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - - if (computationResult.isGraphEmpty()) { - return Stream.empty(); - } - - var nodePropertyValues = computationResult.result() - .map(ClosenessCentralityResult::centralities) - .orElseGet(() -> HugeDoubleArray.newArray(0)) - .asNodeProperties(); - var graph = computationResult.graph(); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> - new CentralityStreamResult( - graph.toOriginalNodeId(nodeId), - nodePropertyValues.doubleValue(nodeId) - )); - - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var nodePropertyValues = result.centralities().asNodeProperties(); + var graph = computationResult.graph(); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> + new CentralityStreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.doubleValue(nodeId) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityNodePropertyValues.java b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityNodePropertyValues.java index 70e481d40d0..2bddf779373 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityNodePropertyValues.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityNodePropertyValues.java @@ -33,10 +33,7 @@ static DegreeCentralityNodePropertyValues from(ComputationResult new DegreeCentralityNodePropertyValues(0, (nodeId) -> -1L)); } - private DegreeCentralityNodePropertyValues( - long nodeCount, - DegreeCentrality.DegreeFunction degreeFunction - ) { + DegreeCentralityNodePropertyValues(long nodeCount, DegreeCentrality.DegreeFunction degreeFunction) { this.nodeCount = nodeCount; this.degreeFunction = degreeFunction; } diff --git a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityStreamSpecification.java b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityStreamSpecification.java index 106c6070afd..8169d0aef8d 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityStreamSpecification.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityStreamSpecification.java @@ -27,7 +27,6 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import java.util.Optional; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -57,12 +56,11 @@ public ComputationResultConsumer runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () -> Optional.ofNullable(computationResult.result()) + () -> computationResult.result() .map(result -> { - var nodePropertyValues = DegreeCentralityNodePropertyValues.from(computationResult); var graph = computationResult.graph(); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) + var nodePropertyValues = new DegreeCentralityNodePropertyValues(graph.nodeCount(), result); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) .filter(nodePropertyValues::hasValue) .mapToObj(nodeId -> new CentralityStreamResult( @@ -70,5 +68,6 @@ public ComputationResultConsumer stream( + public Stream stream( @Name(value = "graphName") String graphName, @Name(value = "configuration", defaultValue = "{}") Map configuration ) { diff --git a/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/CELFStreamSpec.java b/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/CELFStreamSpec.java index aa3fb1a7aac..3974f252d7d 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/CELFStreamSpec.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/CELFStreamSpec.java @@ -26,14 +26,15 @@ import org.neo4j.gds.executor.GdsCallable; import org.neo4j.gds.executor.NewConfigFunction; -import java.util.Optional; +import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.influenceMaximization.CELFStreamProc.DESCRIPTION; @GdsCallable(name = "gds.beta.influenceMaximization.celf.stream", description = DESCRIPTION, executionMode = STREAM) -public class CELFStreamSpec implements AlgorithmSpec, CELFAlgorithmFactory> { +public class CELFStreamSpec implements AlgorithmSpec, CELFAlgorithmFactory> { @Override public String name() { @@ -51,16 +52,20 @@ public NewConfigFunction newConfigFunction() } @Override - public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - return Optional.ofNullable(computationResult.algorithm()) - .map(CELF::resultStream) - .orElseGet(Stream::empty); - }; - + public ComputationResultConsumer> computationResultConsumer() { + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + return LongStream.of(result.keys().toArray()) + .mapToObj(node -> new StreamResult( + graph.toOriginalNodeId(node), + result.getOrDefault(node, 0) + )); + }) + .orElseGet(Stream::empty) + ); } } diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationResult.java b/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/StreamResult.java similarity index 89% rename from algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationResult.java rename to proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/StreamResult.java index abaeca0a80f..212e4e79fad 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/InfluenceMaximizationResult.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/influenceMaximization/StreamResult.java @@ -19,11 +19,11 @@ */ package org.neo4j.gds.influenceMaximization; -public class InfluenceMaximizationResult { +public class StreamResult { public final long nodeId; public final double spread; - InfluenceMaximizationResult(long nodeId, double spread) { + StreamResult(long nodeId, double spread) { this.nodeId = nodeId; this.spread = spread; } diff --git a/proc/centrality/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionStreamSpec.java b/proc/centrality/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionStreamSpec.java index 25f5353a274..5be3aff608f 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionStreamSpec.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/kcore/KCoreDecompositionStreamSpec.java @@ -55,9 +55,9 @@ public ComputationResultConsumer runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () ->computationResult.result() + () -> computationResult.result() .map(result -> { - var coreValues=result.coreValues(); + var coreValues = result.coreValues(); var graph = computationResult.graph(); return LongStream .range(IdMap.START_NODE_ID, graph.nodeCount()) diff --git a/proc/centrality/src/main/java/org/neo4j/gds/pagerank/PageRankStreamSpec.java b/proc/centrality/src/main/java/org/neo4j/gds/pagerank/PageRankStreamSpec.java index ebe87c17e92..6952afdc4be 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/pagerank/PageRankStreamSpec.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/pagerank/PageRankStreamSpec.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.pagerank; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.common.CentralityStreamResult; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -29,6 +30,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.pagerank.PageRankProcCompanion.PAGE_RANK_DESCRIPTION; @@ -52,19 +54,19 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - var graph = computationResult.graph(); - var scores = computationResult.result().get().scores(); - return LongStream.range(0, graph.nodeCount()).mapToObj(nodeId -> { - return new CentralityStreamResult( - graph.toOriginalNodeId(nodeId), - scores.get(nodeId) - ); - }); - }; - + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var scores = result.scores(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(nodeId -> new CentralityStreamResult( + graph.toOriginalNodeId(nodeId), + scores.get(nodeId) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/centrality/src/test/java/org/neo4j/gds/pagerank/ArticleRankProcTest.java b/proc/centrality/src/test/java/org/neo4j/gds/pagerank/ArticleRankProcTest.java index 5f8b890b134..f332bcbd70b 100644 --- a/proc/centrality/src/test/java/org/neo4j/gds/pagerank/ArticleRankProcTest.java +++ b/proc/centrality/src/test/java/org/neo4j/gds/pagerank/ArticleRankProcTest.java @@ -33,7 +33,6 @@ import org.neo4j.gds.scaling.Mean; import org.neo4j.gds.scaling.MinMax; import org.neo4j.gds.scaling.NoneScaler; -import org.neo4j.kernel.impl.core.NodeEntity; import java.util.List; import java.util.Map; @@ -163,27 +162,6 @@ void streamWithSourceNodes() { )); } - @Test - void failOnMissingSourceNodes() { - var sourceNodes = List.of( - new NodeEntity(null, 42), - new NodeEntity(null, 1337) - ); - - var query = GdsCypher.call(GRAPH_NAME) - .algo("articleRank") - .streamMode() - .addPlaceholder("sourceNodes", "sources") - .yields(); - - - assertThatThrownBy(() -> runQuery(query, Map.of("sources", sourceNodes))) - .hasRootCauseExactlyInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Source nodes do not exist in the in-memory graph") - .hasMessageContaining("['1337', '42']"); - } - - @Test void write() { String propertyKey = "pr"; diff --git a/proc/centrality/src/test/java/org/neo4j/gds/pagerank/EigenvectorProcTest.java b/proc/centrality/src/test/java/org/neo4j/gds/pagerank/EigenvectorProcTest.java index ff3b58ebaf5..fa5df4c146e 100644 --- a/proc/centrality/src/test/java/org/neo4j/gds/pagerank/EigenvectorProcTest.java +++ b/proc/centrality/src/test/java/org/neo4j/gds/pagerank/EigenvectorProcTest.java @@ -33,7 +33,6 @@ import org.neo4j.gds.scaling.Mean; import org.neo4j.gds.scaling.MinMax; import org.neo4j.gds.scaling.NoneScaler; -import org.neo4j.kernel.impl.core.NodeEntity; import java.util.List; import java.util.Map; @@ -165,26 +164,6 @@ void streamWithSourceNodes() { )); } - @Test - void failOnMissingSourceNodes() { - var sourceNodes = List.of( - new NodeEntity(null, 42), - new NodeEntity(null, 1337) - ); - - var query = GdsCypher.call(GRAPH_NAME) - .algo("eigenvector") - .streamMode() - .addPlaceholder("sourceNodes", "sources") - .yields(); - - - assertThatThrownBy(() -> runQuery(query, Map.of("sources", sourceNodes))) - .hasRootCauseExactlyInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Source nodes do not exist in the in-memory graph") - .hasMessageContaining("['1337', '42']"); - } - @Test void write() { String propertyKey = "pr"; diff --git a/proc/common/src/main/java/com/neo4j/gds/internal/CustomProceduresUtil.java b/proc/common/src/main/java/com/neo4j/gds/internal/CustomProceduresUtil.java deleted file mode 100644 index a0fd82baf7a..00000000000 --- a/proc/common/src/main/java/com/neo4j/gds/internal/CustomProceduresUtil.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package com.neo4j.gds.internal; - -import org.neo4j.gds.compat.GraphDatabaseApiProxy; -import org.neo4j.internal.kernel.api.exceptions.ProcedureException; -import org.neo4j.kernel.api.procedure.Context; -import org.neo4j.kernel.api.procedure.GlobalProcedures; - -public final class CustomProceduresUtil { - - public static T resolveDependency(Context ctx, Class dependency) { - return GraphDatabaseApiProxy.resolveDependency(ctx.dependencyResolver(), dependency); - } - - public static T lookupComponentProvider(Context ctx, Class component) throws ProcedureException { - var globalProcedures = resolveDependency(ctx, GlobalProcedures.class); - return globalProcedures.lookupComponentProvider(component, false).apply(ctx); - } - - /** - * Like {@link #lookupComponentProvider(Context, Class)} but only allows safe components. - * Safe components are those that are not sandboxed by the kernel and can be used without - * setting the {@code dbms.security.procedures.unrestricted} setting. - */ - public static T lookupSafeComponentProvider(Context ctx, Class component) throws ProcedureException { - var globalProcedures = resolveDependency(ctx, GlobalProcedures.class); - return globalProcedures.lookupComponentProvider(component, true).apply(ctx); - } - - private CustomProceduresUtil() {} -} diff --git a/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index 66587161d4b..0a5eb7ff168 100644 --- a/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/common/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -27,7 +27,7 @@ import org.neo4j.gds.config.ConcurrencyConfig; import org.neo4j.gds.config.ConsecutiveIdsConfig; import org.neo4j.gds.config.SeedConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.nodeproperties.ConsecutiveLongNodePropertyValues; import org.neo4j.gds.nodeproperties.LongIfChangedNodePropertyValues; import org.neo4j.gds.result.CommunityStatistics; @@ -122,7 +122,7 @@ private static LongNodePropertyValues applySizeFilter( var communitySizes = CommunityStatistics.communitySizes( nodeProperties.nodeCount(), nodeProperties::longValue, - Pools.DEFAULT, + DefaultPool.INSTANCE, concurrency ); return new CommunitySizeFilter(nodeProperties, communitySizes, size); diff --git a/proc/common/src/main/java/org/neo4j/gds/GraphStoreUpdater.java b/proc/common/src/main/java/org/neo4j/gds/GraphStoreUpdater.java index c580962461d..422151b5c00 100644 --- a/proc/common/src/main/java/org/neo4j/gds/GraphStoreUpdater.java +++ b/proc/common/src/main/java/org/neo4j/gds/GraphStoreUpdater.java @@ -20,7 +20,7 @@ package org.neo4j.gds; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.huge.FilteredNodePropertyValues; import org.neo4j.gds.core.write.ImmutableNodeProperty; import org.neo4j.gds.core.write.NodeProperty; @@ -39,14 +39,14 @@ public final class GraphStoreUpdater { private GraphStoreUpdater() {} - public static , ALGO_RESULT, CONFIG extends MutatePropertyConfig> void UpdateGraphStore( + public static , ALGO_RESULT, CONFIG extends MutateNodePropertyConfig> void UpdateGraphStore( AbstractResultBuilder resultBuilder, ComputationResult computationResult, ExecutionContext executionContext, final List nodePropertyList ) { var graph = computationResult.graph(); - MutatePropertyConfig mutatePropertyConfig = computationResult.config(); + MutateNodePropertyConfig mutatePropertyConfig = computationResult.config(); var maybeTranslatedProperties = graph .asNodeFilteredGraph() diff --git a/proc/common/src/main/java/org/neo4j/gds/MutateNodePropertyListFunction.java b/proc/common/src/main/java/org/neo4j/gds/MutateNodePropertyListFunction.java index 232caf2580f..04a3e5f8a0c 100644 --- a/proc/common/src/main/java/org/neo4j/gds/MutateNodePropertyListFunction.java +++ b/proc/common/src/main/java/org/neo4j/gds/MutateNodePropertyListFunction.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; /** * A generics adaptation interface? @@ -27,6 +27,6 @@ public interface MutateNodePropertyListFunction< ALGO extends Algorithm, ALGO_RESULT, - CONFIG extends MutatePropertyConfig + CONFIG extends MutateNodePropertyConfig > extends NodePropertyListFunction { } diff --git a/proc/common/src/main/java/org/neo4j/gds/MutatePropertyComputationResultConsumer.java b/proc/common/src/main/java/org/neo4j/gds/MutatePropertyComputationResultConsumer.java index c0c0d7c72bc..34d0205365e 100644 --- a/proc/common/src/main/java/org/neo4j/gds/MutatePropertyComputationResultConsumer.java +++ b/proc/common/src/main/java/org/neo4j/gds/MutatePropertyComputationResultConsumer.java @@ -19,12 +19,12 @@ */ package org.neo4j.gds; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.result.AbstractResultBuilder; -public class MutatePropertyComputationResultConsumer, ALGO_RESULT, CONFIG extends MutatePropertyConfig, RESULT> +public class MutatePropertyComputationResultConsumer, ALGO_RESULT, CONFIG extends MutateNodePropertyConfig, RESULT> extends MutateComputationResultConsumer { private final MutateNodePropertyListFunction nodePropertyListFunction; diff --git a/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java b/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java index ffcb462c93c..1fb2c06ab91 100644 --- a/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java +++ b/proc/common/src/main/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumer.java @@ -23,7 +23,7 @@ import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.config.AlgoBaseConfig; import org.neo4j.gds.config.WritePropertyConfig; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.Capabilities.WriteMode; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -111,7 +111,7 @@ void writeToNeo( .withTerminationFlag(computationResult.algorithm().terminationFlag) .withProgressTracker(progressTracker) .withArrowConnectionInfo(config.arrowConnectionInfo(), computationResult.graphStore().databaseId().databaseName()) - .parallel(Pools.DEFAULT, config.writeConcurrency()) + .parallel(DefaultPool.INSTANCE, config.writeConcurrency()) .build(); try { diff --git a/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java b/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java index 4b99d266282..04b9a5db37d 100644 --- a/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java @@ -27,7 +27,6 @@ import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.DefaultValue; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.NodeLookup; @@ -77,7 +76,7 @@ class MutatePropertyComputationResultConsumerTest { private final ExecutionContext executionContext = ImmutableExecutionContext .builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .log(Neo4jProxy.testLog()) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/common/src/test/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumerTest.java b/proc/common/src/test/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumerTest.java index ee6d39b3c21..2908cf294fe 100644 --- a/proc/common/src/test/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumerTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/WriteNodePropertiesComputationResultConsumerTest.java @@ -23,7 +23,6 @@ import org.neo4j.gds.api.AlgorithmMetaDataSetter; import org.neo4j.gds.api.CloseableResourceRegistry; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.EmptyDependencyResolver; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphCharacteristics; import org.neo4j.gds.api.GraphStore; @@ -84,7 +83,7 @@ class WriteNodePropertiesComputationResultConsumerTest extends BaseTest { private final ExecutionContext executionContext = ImmutableExecutionContext .builder() .databaseId(DatabaseId.from("")) - .dependencyResolver(EmptyDependencyResolver.INSTANCE) + .dependencyResolver(Neo4jProxy.emptyDependencyResolver()) .returnColumns(ProcedureReturnColumns.EMPTY) .log(Neo4jProxy.testLog()) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) diff --git a/proc/community/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutStreamSpec.java index 1d6b82cdbdb..b906d5d94b8 100644 --- a/proc/community/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/approxmaxkcut/ApproxMaxKCutStreamSpec.java @@ -21,7 +21,7 @@ import org.neo4j.gds.CommunityProcCompanion; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.approxmaxkcut.config.ApproxMaxKCutStreamConfig; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -32,6 +32,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.approxmaxkcut.ApproxMaxKCut.APPROX_MAX_K_CUT_DESCRIPTION; import static org.neo4j.gds.executor.ExecutionMode.MUTATE_NODE_PROPERTY; @@ -54,21 +55,23 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) ->{ - if (computationResult.isGraphEmpty()){ - return Stream.empty(); - } - var nodeProperties = CommunityProcCompanion.considerSizeFilter( - computationResult.config(), - computationResult.result() - .map(MaxKCutResult::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) - ); - var graph = computationResult.graph(); - - return LongStream.range(0,graph.nodeCount()) - .filter(nodeProperties::hasValue) - .mapToObj( nodeId -> new StreamResult(graph.toOriginalNodeId(nodeId),nodeProperties.longValue(nodeId))); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodeProperties = CommunityProcCompanion.considerSizeFilter( + computationResult.config(), + result.asNodeProperties() + ); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodeProperties::hasValue) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + nodeProperties.longValue(nodeId) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java index 56f0e18111f..5f73ef899a0 100644 --- a/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/conductance/ConductanceStreamSpec.java @@ -28,6 +28,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.conductance.Conductance.CONDUCTANCE_DESCRIPTION; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @@ -49,22 +50,19 @@ public NewConfigFunction newConfigFunction() { return (username, configuration) -> ConductanceStreamConfig.of(configuration); } - @SuppressWarnings("unchecked") @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - var result=computationResult.result().get(); - var condunctances = result.communityConductances(); - - return LongStream - .range(0, condunctances.capacity()) - .filter(c -> !Double.isNaN(condunctances.get(c))) - .mapToObj(c -> new StreamResult(c, condunctances.get(c))); - }; - + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var condunctances = result.communityConductances(); + return LongStream + .range(0, condunctances.capacity()) + .filter(community -> !Double.isNaN(condunctances.get(community))) + .mapToObj(community -> new StreamResult(community, condunctances.get(community))); + }).orElseGet(Stream::empty) + ); } - } diff --git a/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringSpecificationHelper.java b/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringSpecificationHelper.java index 55629c752ac..f89b2ecc518 100644 --- a/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringSpecificationHelper.java +++ b/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringSpecificationHelper.java @@ -35,13 +35,14 @@ static AbstractResultBuilder computeResult, ProcedureReturnColumns returnColumns ) { + var algorithm=computeResult.algorithm(); if (returnColumns.contains(COLOR_COUNT_FIELD_NAME)) { - procResultBuilder.withColorCount(computeResult.algorithm().usedColors().cardinality()); + procResultBuilder.withColorCount(computeResult.isGraphEmpty() ? 0: algorithm.usedColors().cardinality()); } return procResultBuilder - .withRanIterations(computeResult.isGraphEmpty() ? 0 : computeResult.algorithm().ranIterations()) - .withDidConverge(computeResult.isGraphEmpty() ? false : computeResult.algorithm().didConverge()); + .withRanIterations(computeResult.isGraphEmpty() ? 0 : algorithm.ranIterations()) + .withDidConverge(computeResult.isGraphEmpty() ? false : algorithm.didConverge()); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringStreamSpecification.java index ac89659bd55..f220bc4ca21 100644 --- a/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/k1coloring/K1ColoringStreamSpecification.java @@ -25,7 +25,6 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.executor.AlgorithmSpec; -import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; @@ -58,12 +57,11 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (ComputationResult computationResult, ExecutionContext executionContext) -> - runWithExceptionLogging("Result streaming failed", executionContext.log(), () -> { - if (computationResult.isGraphEmpty()) { - return Stream.empty(); - } - + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { var graph = computationResult.graph(); var config = computationResult.config(); var nodePropertyValues = (NodePropertyValues) CommunityProcCompanion.considerSizeFilter( @@ -79,7 +77,7 @@ public ComputationResultConsumer newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - - return (computationResult, executionContext) -> { - if(computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var result = computationResult.result().get(); - var communities = result.communities(); - var distances = result.distanceFromCenter(); - var silhuette = result.silhouette(); - var graph = computationResult.graph(); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - communities.get(nodeId), - distances.get(nodeId), - silhuette == null ? -1 : silhuette.get(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var communities = result.communities(); + var distances = result.distanceFromCenter(); + var silhuette = result.silhouette(); + var graph = computationResult.graph(); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + communities.get(nodeId), + distances.get(nodeId), + silhuette == null ? -1 : silhuette.get(nodeId) + )); + }).orElseGet(Stream::empty) + ); } - } diff --git a/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java b/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java index aeaa8bcb396..c81841e9275 100644 --- a/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/kmeans/KmeansWriteSpec.java @@ -21,7 +21,7 @@ import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.core.write.NodePropertyExporter; import org.neo4j.gds.executor.AlgorithmSpec; @@ -101,7 +101,7 @@ public ComputationResultConsumer runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () -> { - - return Optional.ofNullable(computationResult.result()) - .map(result -> { - var graph = computationResult.graph(); - var nodePropertyValues = CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(LabelPropagationResult::labels) - .orElseGet(() -> HugeLongArray.newArray(0)) - .asNodeProperties() - ); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - nodePropertyValues.longValue(nodeId) - )); - }).orElseGet(Stream::empty); - } + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodePropertyValues = CommunityProcCompanion.nodeProperties( + computationResult.config(), + result.labels().asNodeProperties() + ); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.longValue(nodeId) + )); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenStreamSpec.java index b73eb68da72..3dd26a591a9 100644 --- a/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenStreamSpec.java @@ -20,11 +20,8 @@ package org.neo4j.gds.leiden; import org.neo4j.gds.CommunityProcCompanion; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; -import org.neo4j.gds.core.utils.paged.HugeLongArray; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; -import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; @@ -33,6 +30,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.leiden.LeidenStreamProc.DESCRIPTION; @@ -55,48 +53,33 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var leidenResult = computationResult.result().get(); - var graph = computationResult.graph(); - var nodeProperties = nodeProperties(computationResult); - - boolean includeIntermediateCommunities = computationResult.config().includeIntermediateCommunities(); - - return LongStream.range(0, graph.nodeCount()) - .filter(nodeProperties::hasValue) - .mapToObj(nodeId -> { - long[] intermediateCommunityIds = includeIntermediateCommunities - ? leidenResult.getIntermediateCommunities(nodeId) - : null; - long communityId = nodeProperties.longValue(nodeId); - - return new StreamResult( - graph.toOriginalNodeId(nodeId), - intermediateCommunityIds, - communityId + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodeProperties = CommunityProcCompanion.nodeProperties( + computationResult.config(), + result.dendrogramManager().getCurrent().asNodeProperties() ); - }); - }; - } + var includeIntermediateCommunities = computationResult.config().includeIntermediateCommunities(); - protected NodePropertyValues nodeProperties(ComputationResult computationResult) { - return getCommunities(computationResult); - } + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodeProperties::hasValue) + .mapToObj(nodeId -> { + long[] intermediateCommunityIds = includeIntermediateCommunities + ? result.getIntermediateCommunities(nodeId) + : null; + long communityId = nodeProperties.longValue(nodeId); - private static NodePropertyValues getCommunities( - ComputationResult computationResult - ) { - return CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(LeidenResult::dendrogramManager) - .map(LeidenDendrogramManager::getCurrent) - .map(HugeLongArray::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) + return new StreamResult( + graph.toOriginalNodeId(nodeId), + intermediateCommunityIds, + communityId + ); + }); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainStreamSpec.java index c579dcc0f70..5c179ccab2b 100644 --- a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainStreamSpec.java @@ -20,11 +20,7 @@ package org.neo4j.gds.louvain; import org.neo4j.gds.CommunityProcCompanion; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; -import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.executor.AlgorithmSpec; -import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; @@ -33,6 +29,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.louvain.LouvainConstants.DESCRIPTION; @@ -55,43 +52,30 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var config = computationResult.config(); + var nodePropertyValues = CommunityProcCompanion.nodeProperties( + config, + result.dendrogramManager().getCurrent().asNodeProperties() + ); + var includeIntermediateCommunities = config.includeIntermediateCommunities(); - var graph = computationResult.graph(); - var nodeCount = graph.nodeCount(); - var nodePropertyValues = nodeProperties(computationResult); - var includeIntermediateCommunities = computationResult.config().includeIntermediateCommunities(); - var louvainResult = computationResult.result().get(); - - return LongStream.range(0, nodeCount) - .boxed(). - filter(nodePropertyValues::hasValue) - .map(nodeId -> { - long[] communities = includeIntermediateCommunities ? louvainResult.getIntermediateCommunities( - nodeId) : null; - long communityId = nodePropertyValues.longValue(nodeId); - return new StreamResult(graph.toOriginalNodeId(nodeId), communities, communityId); - }); - }; - } - - protected NodePropertyValues nodeProperties(ComputationResult computationResult) { - return getCommunities(computationResult); - } - - private static NodePropertyValues getCommunities( - ComputationResult computationResult - ) { - return CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(LouvainResult::dendrogramManager) - .map(LouvainDendrogramManager::getCurrent) - .map(HugeLongArray::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) + return LongStream.range(0, graph.nodeCount()) + .boxed(). + filter(nodePropertyValues::hasValue) + .map(nodeId -> { + var communities = includeIntermediateCommunities + ? result.getIntermediateCommunities(nodeId) + : null; + var communityId = nodePropertyValues.longValue(nodeId); + return new StreamResult(graph.toOriginalNodeId(nodeId), communities, communityId); + }); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/modularity/ModularityStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/modularity/ModularityStreamSpec.java index fd1f80db7dd..c738ef9c7a2 100644 --- a/proc/community/src/main/java/org/neo4j/gds/modularity/ModularityStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/modularity/ModularityStreamSpec.java @@ -28,6 +28,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.modularity.ModularityStreamProc.DESCRIPTION; @@ -50,15 +51,17 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - var modularityResult = computationResult.result() - .orElseGet(ModularityResult::empty); - - var communityModularities = modularityResult.modularityScores(); - return LongStream - .range(0, modularityResult.communityCount()) - .mapToObj(communityModularities::get) - .map(StreamResult::from); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var communityModularities = result.modularityScores(); + return LongStream + .range(0, result.communityCount()) + .mapToObj(communityModularities::get) + .map(StreamResult::from); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationStreamSpecification.java index f2e09f1fdfb..041aaaaa3cd 100644 --- a/proc/community/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationStreamSpecification.java @@ -21,10 +21,7 @@ import org.neo4j.gds.CommunityProcCompanion; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.executor.AlgorithmSpec; -import org.neo4j.gds.executor.ComputationResult; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; import org.neo4j.gds.executor.GdsCallable; @@ -57,33 +54,22 @@ public NewConfigFunction newConfigFunction() @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> - runWithExceptionLogging("Result streaming failed", executionContext.log(), () -> { - if (computationResult.isGraphEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - var config = computationResult.config(); - var nodePropertyValues = nodeProperties(computationResult); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> new ModularityOptimizationStreamResult( - graph.toOriginalNodeId(nodeId), - nodePropertyValues.longValue(nodeId) - )); - }); - } - - private NodePropertyValues nodeProperties( - ComputationResult computationResult - ) { - return CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(ModularityOptimizationResult::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var config = computationResult.config(); + var nodePropertyValues = CommunityProcCompanion.nodeProperties(config, result.asNodeProperties()); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> new ModularityOptimizationStreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.longValue(nodeId) + )); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/scc/SccStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/scc/SccStreamSpec.java index c538f35bc84..28a8fdb42d2 100644 --- a/proc/community/src/main/java/org/neo4j/gds/scc/SccStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/scc/SccStreamSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.scc; -import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -30,6 +30,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.scc.Scc.NOT_VALID; import static org.neo4j.gds.scc.Scc.SCC_DESCRIPTION; @@ -53,17 +54,17 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - Graph graph = computationResult.graph(); - HugeLongArray components = computationResult.result().orElseGet(() -> HugeLongArray.newArray(0)); - - if (graph.isEmpty()) { - return Stream.empty(); - } - - return LongStream.range(0, graph.nodeCount()) - .filter(i -> components.get(i) != NOT_VALID) - .mapToObj(i -> new StreamResult(graph.toOriginalNodeId(i), components.get(i))); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var components = computationResult.result().orElseGet(() -> HugeLongArray.newArray(0)); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(i -> components.get(i) != NOT_VALID) + .mapToObj(i -> new StreamResult(graph.toOriginalNodeId(i), components.get(i))); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientStreamSpec.java b/proc/community/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientStreamSpec.java index 51ae2b5764b..221209ca1f0 100644 --- a/proc/community/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientStreamSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/triangle/LocalClusteringCoefficientStreamSpec.java @@ -28,6 +28,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.triangle.LocalClusteringCoefficientCompanion.DESCRIPTION; @@ -50,19 +51,18 @@ public NewConfigFunction newConfigFuncti @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - - if (computationResult.result().isEmpty()) { - return Stream.of(); - } - - var graph = computationResult.graph(); - var result = computationResult.result().get(); - return LongStream.range(0, graph.nodeCount()) - .mapToObj(i -> new LocalClusteringCoefficientStreamResult( - graph.toOriginalNodeId(i), - result.localClusteringCoefficients().get(i) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + return LongStream.range(0, graph.nodeCount()) + .mapToObj(i -> new LocalClusteringCoefficientStreamResult( + graph.toOriginalNodeId(i), + result.localClusteringCoefficients().get(i) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleCountStatsSpec.java b/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleCountStatsSpec.java index 17e8372425e..ca3b6503de0 100644 --- a/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleCountStatsSpec.java +++ b/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleCountStatsSpec.java @@ -28,6 +28,7 @@ import java.util.stream.Stream; import static org.neo4j.gds.BaseProc.STATS_DESCRIPTION; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STATS; @GdsCallable(name = "gds.triangleCount.stats", description = STATS_DESCRIPTION, executionMode = STATS) @@ -50,21 +51,20 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - var builder = new TriangleCountStatsResult.Builder(); - computationResult.result() - .ifPresent(result -> + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var builder = new TriangleCountStatsResult.Builder(); + builder.withGlobalTriangleCount(result.globalTriangles()); builder - .withGlobalTriangleCount(result.globalTriangles()) - ); - - builder - .withPreProcessingMillis(computationResult.preProcessingMillis()) - .withComputeMillis(computationResult.computeMillis()) - .withNodeCount(computationResult.graph().nodeCount()) - .withConfig(computationResult.config()); - - return Stream.of(builder.build()); - }; + .withPreProcessingMillis(computationResult.preProcessingMillis()) + .withComputeMillis(computationResult.computeMillis()) + .withNodeCount(computationResult.graph().nodeCount()) + .withConfig(computationResult.config()); + return Stream.of(builder.build()); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleStreamSpecification.java index 451c8680983..a469046f79f 100644 --- a/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/triangle/TriangleStreamSpecification.java @@ -27,6 +27,7 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.triangle.TriangleProc.DESCRIPTION; @@ -50,14 +51,14 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer, TriangleCountBaseConfig, Stream> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var resultStream = computationResult.result().get(); - executionContext.closeableResourceRegistry().register(resultStream); - return resultStream; - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + executionContext.closeableResourceRegistry().register(result); + return result; + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamSpecification.java b/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamSpecification.java index 6f7640868db..7f3897ff57d 100644 --- a/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamSpecification.java +++ b/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamSpecification.java @@ -21,7 +21,6 @@ import org.neo4j.gds.CommunityProcCompanion; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.properties.nodes.EmptyLongNodePropertyValues; import org.neo4j.gds.core.utils.paged.dss.DisjointSetStruct; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -33,6 +32,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.wcc.WccSpecification.WCC_DESCRIPTION; @GdsCallable(name = "gds.wcc.stream", description = WCC_DESCRIPTION, executionMode = ExecutionMode.STREAM) @@ -55,26 +55,25 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.isGraphEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - var nodePropertyValues = CommunityProcCompanion.nodeProperties( - computationResult.config(), - computationResult.result() - .map(DisjointSetStruct::asNodeProperties) - .orElse(EmptyLongNodePropertyValues.INSTANCE) - ); - return LongStream - .range(IdMap.START_NODE_ID, graph.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - nodePropertyValues.longValue(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodePropertyValues = CommunityProcCompanion.nodeProperties( + computationResult.config(), + result.asNodeProperties() + ); + return LongStream + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.longValue(nodeId) + )); + }).orElseGet(Stream::empty) + ); } @SuppressWarnings("unused") diff --git a/proc/community/src/test/java/org/neo4j/gds/conductance/ConductanceStreamProcTest.java b/proc/community/src/test/java/org/neo4j/gds/conductance/ConductanceStreamProcTest.java index f4e30d26db8..0d5c6e7b8f4 100644 --- a/proc/community/src/test/java/org/neo4j/gds/conductance/ConductanceStreamProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/conductance/ConductanceStreamProcTest.java @@ -38,7 +38,7 @@ class ConductanceStreamProcTest extends BaseProcTest { static final String GRAPH_NAME = "myGraph"; - @Neo4jGraph + @Neo4jGraph(offsetIds = true) @Language("Cypher") private static final String DB_CYPHER = "CREATE" + diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java index c79e67fdabe..28f9f973e56 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java @@ -37,6 +37,10 @@ static NodePropertyValues nodeProperties(Compu .orElse(EmptyFloatArrayNodePropertyValues.INSTANCE); } + static NodePropertyValues nodeProperties(FastRP.FastRPResult result) { + return new EmbeddingNodePropertyValues(result.embeddings()); + } + private static class EmbeddingNodePropertyValues implements FloatArrayNodePropertyValues { private final HugeObjectArray embeddings; private final long nodeCount; diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamSpec.java index 2eb709448b9..cefe623534f 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPStreamSpec.java @@ -54,19 +54,18 @@ public ComputationResultConsumer runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () -> { - - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - var nodePropertyValues = FastRPCompanion.nodeProperties(computationResult); - return LongStream - .range(IdMap.START_NODE_ID, nodePropertyValues.nodeCount()) - .filter(nodePropertyValues::hasValue) - .mapToObj(nodeId -> new StreamResult(graph.toOriginalNodeId(nodeId), nodePropertyValues.floatArrayValue(nodeId))); - } + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodePropertyValues = FastRPCompanion.nodeProperties(result); + return LongStream + .range(IdMap.START_NODE_ID, nodePropertyValues.nodeCount()) + .filter(nodePropertyValues::hasValue) + .mapToObj(nodeId -> new StreamResult( + graph.toOriginalNodeId(nodeId), + nodePropertyValues.floatArrayValue(nodeId) + )); + }).orElseGet(Stream::empty) ); } } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageStreamSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageStreamSpec.java index 60daa990664..a1b7ce2c1fe 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageStreamSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageStreamSpec.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.embeddings.graphsage; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.embeddings.graphsage.algo.GraphSage; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageAlgorithmFactory; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageResult; @@ -60,23 +61,19 @@ public ComputationResultConsumer runWithExceptionLogging( "GraphSage streaming failed", executionContext.log(), - () -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - var graph = computationResult.graph(); - var embeddings = computationResult.result().get().embeddings(); - - return LongStream.range(0, graph.nodeCount()) - .mapToObj(internalNodeId -> new StreamResult( - graph.toOriginalNodeId(internalNodeId), - embeddings.get(internalNodeId) - )); - } + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var embeddings = result.embeddings(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(internalNodeId -> new StreamResult( + graph.toOriginalNodeId(internalNodeId), + embeddings.get(internalNodeId) + )); + }).orElseGet(Stream::empty) ); } - @Override public ValidationConfiguration validationConfig(ExecutionContext executionContext) { return new GraphSageConfigurationValidation<>(executionContext.modelCatalog()); diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainSpec.java index 4329b20f5cb..df8e24dfaf3 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.embeddings.graphsage; -import org.neo4j.gds.compat.ProxyUtil; +import org.neo4j.gds.compat.GdsVersionInfoProvider; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageTrain; import org.neo4j.gds.embeddings.graphsage.algo.GraphSageTrainAlgorithmFactory; @@ -50,7 +50,7 @@ public String name() { @Override public GraphSageTrainAlgorithmFactory algorithmFactory(ExecutionContext executionContext) { - var gdsVersion = ProxyUtil.GDS_VERSION_INFO.gdsVersion(); + var gdsVersion = GdsVersionInfoProvider.GDS_VERSION_INFO.gdsVersion(); return new GraphSageTrainAlgorithmFactory(gdsVersion); } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNStreamSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNStreamSpec.java index fd7f748ecfa..c97adaf3b7b 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNStreamSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNStreamSpec.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.embeddings.hashgnn; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -52,7 +53,6 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> runWithExceptionLogging( "HashGNN streaming failed", executionContext.log(), @@ -60,7 +60,7 @@ public ComputationResultConsumer { var graph = computationResult.graph(); return LongStream - .range(0, graph.nodeCount()) + .range(IdMap.START_NODE_ID, graph.nodeCount()) .mapToObj(i -> new StreamResult( graph.toOriginalNodeId(i), result.embeddings().doubleArrayValue(i) diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecStreamSpec.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecStreamSpec.java index eb72c55a8d5..b8c8afaee0c 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecStreamSpec.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/node2vec/Node2VecStreamSpec.java @@ -29,6 +29,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.beta.node2vec.stream", description = Node2VecCompanion.DESCRIPTION, executionMode = STREAM) @@ -50,7 +51,10 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> computationResult.result() + return (computationResult, executionContext) -> runWithExceptionLogging( + "Node2Vec streaming failed", + executionContext.log(), + () -> computationResult.result() .map(result -> { var graph = computationResult.graph(); var nodePropertyValues = new EmbeddingNodePropertyValues(result.embeddings()); @@ -61,7 +65,6 @@ public ComputationResultConsumer { assertThat(row.get("embedding")) diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/ApproximateLinkPrediction.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/ApproximateLinkPrediction.java index cf45603fe3d..205d1d1b2d7 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/ApproximateLinkPrediction.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/ApproximateLinkPrediction.java @@ -20,7 +20,7 @@ package org.neo4j.gds.ml.linkmodels.pipeline.predict; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -85,7 +85,7 @@ LinkPredictionResult predictLinks(LinkPredictionSimilarityComputer linkPredictio targetNodeFilter ), ImmutableKnnContext.of( - Pools.DEFAULT, + DefaultPool.INSTANCE, progressTracker ) ); diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateResultConsumer.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateResultConsumer.java index 4e7ff197c1e..728bb50844d 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateResultConsumer.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateResultConsumer.java @@ -25,8 +25,8 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.ResultBuilderFunction; import org.neo4j.gds.core.Aggregation; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.executor.ComputationResult; @@ -65,7 +65,7 @@ protected void updateGraphStore( .orientation(Orientation.UNDIRECTED) .addPropertyConfig(GraphFactory.PropertyConfig.of(computationResult.config().mutateProperty())) .concurrency(concurrency) - .executorService(Pools.DEFAULT) + .executorService(DefaultPool.INSTANCE) .build(); var resultWithHistogramBuilder = (MutateResult.Builder) resultBuilder; diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineStreamSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineStreamSpec.java index 083db6f7294..18d18b1219c 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineStreamSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineStreamSpec.java @@ -30,6 +30,7 @@ import java.util.Collection; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.ml.linkmodels.pipeline.LinkPredictionPipelineCompanion.PREDICT_DESCRIPTION; @@ -62,19 +63,22 @@ public NewConfigFunction newConfigFun @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - return computationResult.result().map(result -> { - var graphStore = computationResult.graphStore(); - Collection labelFilter = computationResult.algorithm().labelFilter().predictNodeLabels(); - var graph = graphStore.getGraph(labelFilter); + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graphStore = computationResult.graphStore(); + Collection labelFilter = computationResult.algorithm().labelFilter().predictNodeLabels(); + var graph = graphStore.getGraph(labelFilter); - return result.stream() - .map(predictedLink -> new StreamResult( - graph.toOriginalNodeId(predictedLink.sourceId()), - graph.toOriginalNodeId(predictedLink.targetId()), - predictedLink.probability() - )); - }).orElseGet(Stream::empty); - }; + return result.stream() + .map(predictedLink -> new StreamResult( + graph.toOriginalNodeId(predictedLink.sourceId()), + graph.toOriginalNodeId(predictedLink.targetId()), + predictedLink.probability() + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java index 35e5170ac1e..f7705b922e2 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineMutateConfig.java @@ -21,13 +21,15 @@ import org.immutables.value.Value; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.config.MutatePropertyConfig; import org.neo4j.gds.config.MutateRelationshipConfig; +import org.neo4j.gds.config.MutateRelationshipPropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; @Configuration @SuppressWarnings("immutables:subtype") -public interface LinkPredictionPredictPipelineMutateConfig extends LinkPredictionPredictPipelineBaseConfig, MutateRelationshipConfig, MutatePropertyConfig { +public interface LinkPredictionPredictPipelineMutateConfig extends LinkPredictionPredictPipelineBaseConfig, + MutateRelationshipConfig, + MutateRelationshipPropertyConfig { @Value.Default @Override default String mutateProperty() { diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResult.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResult.java index cf6e20e4c11..e1552d0269c 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResult.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/MutateResult.java @@ -90,7 +90,9 @@ void recordHistogramValue(double value) { return; } - histogram.recordValue(value); + //HISTOGRAM_PRECISION_DEFAULT hence numberOfSignificantValueDigits is 1E-5, so it can't separate 0 and 1E-5 + //Therefore we can floor at 1E-6 and smaller probabilities between 0 and 1E-6 is unnecessary. + if (value >= 1E-6) histogram.recordValue(value); else histogram.recordValue(1E-6); } Builder withSamplingStats(Map samplingStats) { diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainSpec.java index 198bd05e691..5eef869c939 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainSpec.java @@ -20,7 +20,7 @@ package org.neo4j.gds.ml.linkmodels.pipeline.train; import org.neo4j.gds.VerifyThatModelCanBeStored; -import org.neo4j.gds.compat.ProxyUtil; +import org.neo4j.gds.compat.GdsVersionInfoProvider; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -53,7 +53,7 @@ public String name() { @Override public LinkPredictionTrainPipelineAlgorithmFactory algorithmFactory(ExecutionContext executionContext) { - var gdsVersion = ProxyUtil.GDS_VERSION_INFO.gdsVersion(); + var gdsVersion = GdsVersionInfoProvider.GDS_VERSION_INFO.gdsVersion(); return new LinkPredictionTrainPipelineAlgorithmFactory(executionContext, gdsVersion); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineStreamSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineStreamSpec.java index 42ea60d46b5..ba56c36360e 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineStreamSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineStreamSpec.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.ml.pipeline.node.classification.predict; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.executor.AlgorithmSpec; @@ -62,33 +63,27 @@ public NewConfigFunction newConfi @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> - runWithExceptionLogging( - "Result streaming failed", - executionContext.log(), - () -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { var pipelineGraphFilter = computationResult.algorithm().nodePropertyStepFilter(); var graph = computationResult.graphStore().getGraph(pipelineGraphFilter.nodeLabels()); - var result = computationResult.result().get(); var predictedClasses = result.predictedClasses(); var predictedProbabilities = result.predictedProbabilities(); return LongStream - .range(0, graph.nodeCount()) - .boxed() - .map(nodeId -> + .range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(nodeId -> new NodeClassificationStreamResult( graph.toOriginalNodeId(nodeId), predictedClasses.get(nodeId), nodePropertiesAsList(predictedProbabilities, nodeId) ) ); - } - ); + }).orElseGet(Stream::empty) + ); } private static List nodePropertiesAsList( diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainSpec.java index fe8ab7f80d7..c7d1fa36a4a 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainSpec.java @@ -20,7 +20,7 @@ package org.neo4j.gds.ml.pipeline.node.classification.predict; import org.neo4j.gds.VerifyThatModelCanBeStored; -import org.neo4j.gds.compat.ProxyUtil; +import org.neo4j.gds.compat.GdsVersionInfoProvider; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResult; @@ -52,7 +52,7 @@ public String name() { @Override public NodeClassificationTrainPipelineAlgorithmFactory algorithmFactory(ExecutionContext executionContext) { - return new NodeClassificationTrainPipelineAlgorithmFactory(executionContext, ProxyUtil.GDS_VERSION_INFO.gdsVersion()); + return new NodeClassificationTrainPipelineAlgorithmFactory(executionContext, GdsVersionInfoProvider.GDS_VERSION_INFO.gdsVersion()); } @Override diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPredictPipelineMutateConfig.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPredictPipelineMutateConfig.java index 77ac8d862a4..12c1c5c7d35 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPredictPipelineMutateConfig.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPredictPipelineMutateConfig.java @@ -24,7 +24,7 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.config.MutatePropertyConfig; +import org.neo4j.gds.config.MutateNodePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; import java.util.Collection; @@ -33,7 +33,9 @@ @Configuration @SuppressWarnings("immutables:subtype") -public interface NodeClassificationPredictPipelineMutateConfig extends NodeClassificationPredictPipelineMutateOrWriteConfig, MutatePropertyConfig { +public interface NodeClassificationPredictPipelineMutateConfig extends + NodeClassificationPredictPipelineMutateOrWriteConfig, + MutateNodePropertyConfig { static NodeClassificationPredictPipelineMutateConfig of(String username, CypherMapWrapper config) { return new NodeClassificationPredictPipelineMutateConfigImpl(username, config); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainSpec.java index 5c36952306c..ed04db2d0b1 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainSpec.java @@ -20,7 +20,7 @@ package org.neo4j.gds.ml.pipeline.node.regression; import org.neo4j.gds.VerifyThatModelCanBeStored; -import org.neo4j.gds.compat.ProxyUtil; +import org.neo4j.gds.compat.GdsVersionInfoProvider; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -54,7 +54,7 @@ public String name() { @Override public NodeRegressionTrainPipelineAlgorithmFactory algorithmFactory(ExecutionContext executionContext) { - var gdsVersion = ProxyUtil.GDS_VERSION_INFO.gdsVersion(); + var gdsVersion = GdsVersionInfoProvider.GDS_VERSION_INFO.gdsVersion(); return new NodeRegressionTrainPipelineAlgorithmFactory(executionContext, gdsVersion); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineStreamSpec.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineStreamSpec.java index 5d90435431e..5772d3641a7 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineStreamSpec.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineStreamSpec.java @@ -76,8 +76,8 @@ public ComputationResultConsumer { - return computationResult.result().map(result -> { + () -> computationResult.result() + .map(result -> { Graph graph = computationResult.graph(); NodePropertyValues nodePropertyValues = result.asNodeProperties(); return LongStream @@ -87,8 +87,7 @@ public ComputationResultConsumer. + */ +package org.neo4j.gds.ml.linkmodels.pipeline.predict; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class MutateResultTest { + + @Test + void shouldRecordResults() { + var resultWithHistogramBuilder = new MutateResult.Builder().withHistogram(); + assertThatNoException().isThrownBy(() -> { + resultWithHistogramBuilder.recordHistogramValue(0.9); + resultWithHistogramBuilder.recordHistogramValue(5E-10); + resultWithHistogramBuilder.recordHistogramValue(5E-20); + }); + } + +} diff --git a/proc/misc/build.gradle b/proc/misc/build.gradle index 3b19237f04b..7c04a15cff9 100644 --- a/proc/misc/build.gradle +++ b/proc/misc/build.gradle @@ -5,6 +5,8 @@ description = 'Neo4j Graph Data Science :: Procedures :: Misc' group = 'org.neo4j.gds' dependencies { + annotationProcessor project(':procedure-collector') + implementation project(':algo-common') implementation project(':annotations') implementation project(':config-api') diff --git a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java index b733d740082..6ceae741a78 100644 --- a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java +++ b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesMutateProc.java @@ -26,6 +26,7 @@ import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.gds.results.StandardMutateResult; import org.neo4j.procedure.Description; +import org.neo4j.procedure.Internal; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -65,6 +66,8 @@ public Stream estimate( ).computeEstimate(graphName, configuration); } + @Internal + @Deprecated(forRemoval = true) @Procedure(value = "gds.alpha.scaleProperties.mutate", deprecatedBy = "gds.scaleProperties.mutate") @Description(SCALE_PROPERTIES_DESCRIPTION) public Stream alphaMutate( diff --git a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java index 37212baae9d..5a876d62d83 100644 --- a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java +++ b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java @@ -40,17 +40,11 @@ static NodePropertyValues nodeProperties( .map(ScaleProperties.Result::scaledProperties) .orElseGet(() -> HugeObjectArray.newArray(double[].class, 0)); - return new DoubleArrayNodePropertyValues() { - @Override - public long nodeCount() { - return size; - } - - @Override - public double[] doubleArrayValue(long nodeId) { - return scaledProperties.get(nodeId); - } - }; + return new ScaledNodePropertyValues(size, scaledProperties); + } + + static NodePropertyValues nodeProperties(long size, HugeObjectArray scaledProperties) { + return new ScaledNodePropertyValues(size, scaledProperties); } static void validateLegacyScalers(ScalePropertiesBaseConfig config, boolean allowL1L2Scalers) { @@ -59,4 +53,25 @@ static void validateLegacyScalers(ScalePropertiesBaseConfig config, boolean allo ScalerFactory.throwForInvalidScaler(specifiedScaler); } } + + static final class ScaledNodePropertyValues implements DoubleArrayNodePropertyValues { + + private final long size; + private final HugeObjectArray scaledProperties; + + private ScaledNodePropertyValues(long size, HugeObjectArray scaledProperties) { + this.size = size; + this.scaledProperties = scaledProperties; + } + + @Override + public long nodeCount() { + return size; + } + + @Override + public double[] doubleArrayValue(long nodeId) { + return scaledProperties.get(nodeId); + } + } } diff --git a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java index 5c69ff038a0..71bf380cd56 100644 --- a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java +++ b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamProc.java @@ -24,6 +24,7 @@ import org.neo4j.gds.executor.ProcedureExecutor; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; +import org.neo4j.procedure.Internal; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -65,6 +66,8 @@ public Stream estimate( ).computeEstimate(graphName, configuration); } + @Internal + @Deprecated(forRemoval = true) @Procedure(value = "gds.alpha.scaleProperties.stream", deprecatedBy = "gds.scaleProperties.stream") @Description(SCALE_PROPERTIES_DESCRIPTION) public Stream alphaStream( diff --git a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamSpec.java b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamSpec.java index 54b61ef29ec..c3885533f85 100644 --- a/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamSpec.java +++ b/proc/misc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesStreamSpec.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.scaling; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -32,6 +31,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; import static org.neo4j.gds.scaling.ScalePropertiesProc.SCALE_PROPERTIES_DESCRIPTION; import static org.neo4j.gds.scaling.ScalePropertiesProc.validateLegacyScalers; @@ -66,21 +66,20 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - - NodePropertyValues nodeProperties = ScalePropertiesProc.nodeProperties(computationResult); - - return LongStream - .range(0, graph.nodeCount()) - .mapToObj(nodeId -> new ScalePropertiesStreamProc.Result( - graph.toOriginalNodeId(nodeId), - nodeProperties.doubleArrayValue(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var nodeProperties = ScalePropertiesProc.nodeProperties(graph.nodeCount(), result.scaledProperties()); + return LongStream + .range(0, graph.nodeCount()) + .mapToObj(nodeId -> new ScalePropertiesStreamProc.Result( + graph.toOriginalNodeId(nodeId), + nodeProperties.doubleArrayValue(nodeId) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java index fe277bc2b42..4eaa80bda84 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java @@ -28,31 +28,34 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; + public final class ShortestPathStreamResultConsumer, CONFIG extends AlgoBaseConfig> implements ComputationResultConsumer> { @Override public Stream consume( - ComputationResult computationResult, ExecutionContext executionContext + ComputationResult computationResult, + ExecutionContext executionContext ) { - - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var graph = computationResult.graph(); - var shouldReturnPath = executionContext - .returnColumns() - .contains("path") && computationResult.graphStore().capabilities().canWriteToDatabase(); - - var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()); - - var resultStream = computationResult.result().get() - .mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); - - // this is necessary in order to close the result stream which triggers - // the progress tracker to close its root task - executionContext.closeableResourceRegistry().register(resultStream); - - return resultStream; + return runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var shouldReturnPath = executionContext.returnColumns().contains("path") + && computationResult.graphStore().capabilities().canWriteToDatabase(); + + var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()); + + var resultStream = result.mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); + + // this is necessary in order to close the result stream which triggers + // the progress tracker to close its root task + executionContext.closeableResourceRegistry().register(resultStream); + + return resultStream; + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/all/AllShortestPathsStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/all/AllShortestPathsStreamSpec.java index 399a4fcbbbe..89af4b524a3 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/all/AllShortestPathsStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/all/AllShortestPathsStreamSpec.java @@ -27,7 +27,7 @@ import org.neo4j.gds.allshortestpaths.MSBFSAllShortestPaths; import org.neo4j.gds.allshortestpaths.WeightedAllShortestPaths; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; @@ -64,14 +64,14 @@ public MSBFSASPAlgorithm build( if (configuration.hasRelationshipWeightProperty()) { return new WeightedAllShortestPaths( graph, - Pools.DEFAULT, + DefaultPool.INSTANCE, configuration.concurrency() ); } else { return new MSBFSAllShortestPaths( graph, configuration.concurrency(), - Pools.DEFAULT + DefaultPool.INSTANCE ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamSpec.java index 615725fc664..f1dd4c8096e 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamSpec.java @@ -37,6 +37,7 @@ import java.util.function.Function; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @@ -60,31 +61,24 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer, RandomWalkStreamConfig, Stream> computationResultConsumer() { return (computationResult, executionContext) -> { - var graph = computationResult.graph(); - if (graph.isEmpty()) { - return Stream.empty(); - } - - var returnPath = executionContext - .returnColumns() - .contains("path"); - + var returnPath = executionContext.returnColumns().contains("path"); Function, Path> pathCreator = returnPath - ? (List nodes) -> PathFactory.create( - executionContext.nodeLookup(), - nodes, - RelationshipType.withName("NEXT") - ) + ? (List nodes) -> PathFactory.create(executionContext.nodeLookup(), nodes, RelationshipType.withName("NEXT")) : (List nodes) -> null; - - return computationResult.result() - .orElseGet(Stream::empty) - .map(nodes -> { - var translatedNodes = translateInternalToNeoIds(nodes, graph); - var path = pathCreator.apply(translatedNodes); - - return new StreamResult(translatedNodes, path); - }); + return runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + return result + .map(nodes -> { + var translatedNodes = translateInternalToNeoIds(nodes, graph); + var path = pathCreator.apply(translatedNodes); + return new StreamResult(translatedNodes, path); + }); + }).orElseGet(Stream::empty) + ); }; } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/bellmanford/BellmanFordStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/bellmanford/BellmanFordStreamSpec.java index 55a62579726..b3d119704e8 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/bellmanford/BellmanFordStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/bellmanford/BellmanFordStreamSpec.java @@ -32,6 +32,7 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.bellmanFord.stream", description = BellmanFord.DESCRIPTION, executionMode = STREAM) @@ -55,40 +56,37 @@ public NewConfigFunction newConfigFunction() { @SuppressWarnings("unchecked") @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - - var graph = computationResult.graph(); - - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var shouldReturnPath = executionContext - .returnColumns() - .contains("route") - && computationResult.graphStore().capabilities().canWriteToDatabase(); - - var result = computationResult.result().get(); - var containsNegativeCycle = result.containsNegativeCycle(); - - var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()) - .withIsCycle(containsNegativeCycle); - - PathFindingResult algorithmResult; - if (containsNegativeCycle) { - algorithmResult = result.negativeCycles(); - } else { - algorithmResult = result.shortestPaths(); - } - - var resultStream = algorithmResult.mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); - - // this is necessary in order to close the result stream which triggers - // the progress tracker to close its root task - executionContext.closeableResourceRegistry().register(resultStream); - return resultStream; - - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var shouldReturnPath = executionContext + .returnColumns() + .contains("route") + && computationResult.graphStore().capabilities().canWriteToDatabase(); + + var containsNegativeCycle = result.containsNegativeCycle(); + + var resultBuilder = new StreamResult.Builder(graph, executionContext.nodeLookup()) + .withIsCycle(containsNegativeCycle); + + PathFindingResult algorithmResult; + if (containsNegativeCycle) { + algorithmResult = result.negativeCycles(); + } else { + algorithmResult = result.shortestPaths(); + } + + var resultStream = algorithmResult.mapPaths(path -> resultBuilder.build(path, shouldReturnPath)); + + // this is necessary in order to close the result stream which triggers + // the progress tracker to close its root task + executionContext.closeableResourceRegistry().register(resultStream); + return resultStream; + }).orElseGet(Stream::empty) + ); } @Override diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java index f4903e53aad..3e843ba1ed0 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/singlesource/delta/AllShortestPathsDeltaStreamSpec.java @@ -53,13 +53,12 @@ public NewConfigFunction newConfigFunction() return (username, configuration) -> AllShortestPathsDeltaStreamConfig.of(configuration); } - @SuppressWarnings("unchecked") @Override public ComputationResultConsumer> computationResultConsumer() { return new ShortestPathStreamResultConsumer<>(); } -@Override + @Override public boolean releaseProgressTask() { return false; } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeStreamSpec.java index bb1f0dcf944..804dc997511 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeStreamSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.paths.spanningtree; -import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -33,6 +33,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.beta.spanningTree.stream", description = SpanningTreeWriteProc.DESCRIPTION, executionMode = STREAM) @@ -55,24 +56,24 @@ public NewConfigFunction newConfigFunction() { } public ComputationResultConsumer> computationResultConsumer() { - - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var sourceNode = computationResult.config().sourceNode(); - Graph graph = computationResult.graph(); - SpanningTree spanningTree = computationResult.result().get(); - return LongStream.range(0, graph.nodeCount()) - .filter(nodeId -> spanningTree.parent(nodeId) >= 0 || sourceNode == graph.toOriginalNodeId(nodeId)) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - (sourceNode == graph.toOriginalNodeId(nodeId)) ? - sourceNode : - graph.toOriginalNodeId(spanningTree.parent(nodeId)), - spanningTree.costToParent(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var sourceNode = computationResult.config().sourceNode(); + var graph = computationResult.graph(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodeId -> result.parent(nodeId) >= 0 || sourceNode == graph.toOriginalNodeId(nodeId)) + .mapToObj(nodeId -> { + var originalId = graph.toOriginalNodeId(nodeId); + return new StreamResult( + originalId, + (sourceNode == originalId) ? sourceNode : graph.toOriginalNodeId(result.parent(nodeId)), + result.costToParent(nodeId) + ); + }); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/steiner/SteinerTreeStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/steiner/SteinerTreeStreamSpec.java index fb228f3d92f..0b3c7bcbb72 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/steiner/SteinerTreeStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/steiner/SteinerTreeStreamSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.paths.steiner; -import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -33,6 +33,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.beta.SteinerTree.stream", description = SteinerTreeStatsProc.DESCRIPTION, executionMode = STREAM) @@ -55,26 +56,26 @@ public NewConfigFunction newConfigFunction() { } public ComputationResultConsumer> computationResultConsumer() { - - return (computationResult, executionContext) -> { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - var sourceNode = computationResult.config().sourceNode(); - Graph graph = computationResult.graph(); - var steinerTreeResult = computationResult.result().get(); - var parentArray = steinerTreeResult.parentArray(); - var costArray = steinerTreeResult.relationshipToParentCost(); - return LongStream.range(0, graph.nodeCount()) - .filter(nodeId -> parentArray.get(nodeId) != ShortestPathsSteinerAlgorithm.PRUNED) - .mapToObj(nodeId -> new StreamResult( - graph.toOriginalNodeId(nodeId), - (sourceNode == graph.toOriginalNodeId(nodeId)) ? - sourceNode : - graph.toOriginalNodeId(parentArray.get(nodeId)), - costArray.get(nodeId) - )); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var sourceNode = computationResult.config().sourceNode(); + var graph = computationResult.graph(); + var parents = result.parentArray(); + var costs = result.relationshipToParentCost(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .filter(nodeId -> parents.get(nodeId) != ShortestPathsSteinerAlgorithm.PRUNED) + .mapToObj(nodeId -> { + var originalId = graph.toOriginalNodeId(nodeId); + return new StreamResult( + originalId, + (sourceNode == originalId) ? sourceNode : graph.toOriginalNodeId(parents.get(nodeId)), + costs.get(nodeId) + ); + }); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/topologicalsort/TopologicalSortStreamSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/topologicalsort/TopologicalSortStreamSpec.java index 822d1e84651..95861f1ba98 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/topologicalsort/TopologicalSortStreamSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/topologicalsort/TopologicalSortStreamSpec.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.paths.topologicalsort; -import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.IdMap; import org.neo4j.gds.executor.AlgorithmSpec; import org.neo4j.gds.executor.ComputationResultConsumer; import org.neo4j.gds.executor.ExecutionContext; @@ -33,6 +33,7 @@ import java.util.stream.LongStream; import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.executor.ExecutionMode.STREAM; @GdsCallable(name = "gds.alpha.topologicalSort.stream", description = TopologicalSortStreamProc.TOPOLOGICAL_SORT_DESCRIPTION, executionMode = STREAM) @@ -55,16 +56,18 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> - { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - Graph graph = computationResult.graph(); - var topologicallySortedNodes = computationResult.result().get().value(); - return LongStream.range(0, graph.nodeCount()) - .mapToObj(nodeId -> new TopologicalSortStreamResult(graph.toOriginalNodeId(topologicallySortedNodes.get(nodeId)))); - }; + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var topologicallySortedNodes = result.value(); + return LongStream.range(IdMap.START_NODE_ID, graph.nodeCount()) + .mapToObj(nodeId -> new TopologicalSortStreamResult( + graph.toOriginalNodeId(topologicallySortedNodes.get(nodeId)) + )); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumer.java index f92ed5b59a4..d8c5ccb455f 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumer.java @@ -26,6 +26,8 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; + class BfsStreamComputationResultConsumer implements ComputationResultConsumer> { private final PathFactoryFacade pathFactoryFacade; @@ -39,20 +41,20 @@ public Stream consume( ComputationResult computationResult, ExecutionContext executionContext ) { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - return TraverseStreamComputationResultConsumer.consume( - computationResult.config().sourceNode(), - computationResult.result().get(), - computationResult.graph()::toOriginalNodeId, - computationResult.graph().isEmpty(), - BfsStreamResult::new, - executionContext.returnColumns().contains("path"), - pathFactoryFacade, - BfsStreamProc.NEXT, - executionContext.nodeLookup() + return runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> TraverseStreamComputationResultConsumer.consume( + computationResult.config().sourceNode(), + result, + computationResult.graph()::toOriginalNodeId, + BfsStreamResult::new, + executionContext.returnColumns().contains("path"), + pathFactoryFacade, + BfsStreamProc.NEXT, + executionContext.nodeLookup() + )).orElseGet(Stream::empty) ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumer.java index a3c70f64d67..15c75e73438 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumer.java @@ -26,6 +26,8 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; + class DfsStreamComputationResultConsumer implements ComputationResultConsumer> { private final PathFactoryFacade pathFactoryFacade; @@ -39,20 +41,20 @@ public Stream consume( ComputationResult computationResult, ExecutionContext executionContext ) { - if (computationResult.result().isEmpty()) { - return Stream.empty(); - } - - return TraverseStreamComputationResultConsumer.consume( - computationResult.config().sourceNode(), - computationResult.result().get(), - computationResult.graph()::toOriginalNodeId, - computationResult.graph().isEmpty(), - DfsStreamResult::new, - executionContext.returnColumns().contains("path"), - pathFactoryFacade, - DfsStreamProc.NEXT, - executionContext.nodeLookup() + return runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> TraverseStreamComputationResultConsumer.consume( + computationResult.config().sourceNode(), + result, + computationResult.graph()::toOriginalNodeId, + DfsStreamResult::new, + executionContext.returnColumns().contains("path"), + pathFactoryFacade, + DfsStreamProc.NEXT, + executionContext.nodeLookup() + )).orElseGet(Stream::empty) ); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumer.java index acf177cb0e5..91528d371bd 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumer.java @@ -37,19 +37,14 @@ private TraverseStreamComputationResultConsumer() {} static Stream consume( long sourceNodeId, - @Nullable HugeLongArray nodes, + HugeLongArray nodes, LongUnaryOperator toOriginalNodeId, - boolean graphIsEmpty, ConcreteResultTransformer resultTransformer, boolean shouldReturnPath, PathFactoryFacade pathFactoryFacade, RelationshipType relationshipType, NodeLookup nodeLookup ) { - if (graphIsEmpty || null == nodes) { - return Stream.empty(); - } - var nodesArray = nodes.toArray(); var nodeList = Arrays .stream(nodesArray) diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java index 9196c151908..0327691896d 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/kspanningtree/KSpanningTreeWriteProcTest.java @@ -38,7 +38,7 @@ class KSpanningTreeWriteProcTest extends BaseProcTest { private static final String GRAPH_NAME = "graph"; - @Neo4jGraph + @Neo4jGraph(offsetIds = true) private static final String DB_CYPHER = "CREATE (a:Node {name:'a'})\n" + "CREATE (b:Node {name:'b'})\n" + diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamProcTest.java index 07fe99d3c52..a57707d9da0 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/randomwalk/RandomWalkStreamProcTest.java @@ -26,7 +26,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.TestSupport; import org.neo4j.gds.catalog.GraphProjectProc; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.graphdb.Path; @@ -128,7 +128,7 @@ void shouldReturnPath() { @Test void shouldStopWhenStreamIsNotLongerConsumed() { - var pool = (ThreadPoolExecutor) Pools.DEFAULT; + var pool = (ThreadPoolExecutor) DefaultPool.INSTANCE; assumeThat(pool.getActiveCount()).as("Test requires that no other threads are currently running").isEqualTo(0); // re-setup with a larger graph @@ -191,34 +191,7 @@ void shouldStopWhenStreamIsNotLongerConsumed() { // we're done or fail the test assertThat(pool.getActiveCount()).isEqualTo(0); } - - @Test - void shouldThrowOnUnknownStartNode() { - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("gds", "randomWalk") - .streamMode() - .addParameter("walksPerNode", 3) - .addParameter("walkLength", 10) - .addParameter("sourceNodes", 42) - .yields(); - - assertError(query, "Source nodes do not exist in the in-memory graph: ['42']"); - } - - @Test - void shouldThrowOnUnselectedStartNode() { - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("gds", "randomWalk") - .streamMode() - .addParameter("walksPerNode", 3) - .addParameter("walkLength", 10) - .addParameter("sourceNodes", 3) - .addParameter("nodeLabels", List.of("Node1", "Node2")) - .yields(); - - assertError(query, "Source nodes do not exist in the in-memory graph for the labels ['Node1', 'Node2']: ['3']"); - } - + @Test void shouldRunMemoryEstimation() { String query = GdsCypher.call(DEFAULT_GRAPH_NAME) diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java index 91872423e2d..b0dfeeeebef 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensStreamProcTest.java @@ -200,7 +200,7 @@ void testStream() { assertCypherResult(query, expected); }); } - + @AfterEach void teardown() { GraphStoreCatalog.removeAllLoadedGraphs(); diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumerTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumerTest.java index c11f0d4b7b9..f47887892a1 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumerTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamComputationResultConsumerTest.java @@ -54,7 +54,6 @@ class BfsStreamComputationResultConsumerTest { @Test void shouldNotComputePath() { - when(graphMock.isEmpty()).thenReturn(false); when(graphMock.toOriginalNodeId(anyLong())).then(returnsFirstArg()); when(computationResultMock.graph()).thenReturn(graphMock); @@ -83,7 +82,6 @@ void shouldNotComputePath() { @Test void shouldComputePath() { - when(graphMock.isEmpty()).thenReturn(false); when(graphMock.toOriginalNodeId(anyLong())).then(returnsFirstArg()); when(computationResultMock.graph()).thenReturn(graphMock); diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java index 750c772d584..37968f03d05 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/BfsStreamProcTest.java @@ -229,22 +229,10 @@ void failOnInvalidSourceNode() { String query = GdsCypher.call(DEFAULT_GRAPH_NAME) .algo("bfs") .streamMode() - .addParameter("sourceNode", 42) + .addParameter("sourceNode", 4242) .yields(); - assertError(query, "Source node does not exist in the in-memory graph: `42`"); + assertError(query, "Source node does not exist in the in-memory graph: `4242"); } - @Test - void failOnInvalidEndNode() { - loadCompleteGraph(DEFAULT_GRAPH_NAME); - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("bfs") - .streamMode() - .addParameter("sourceNode", 0) - .addParameter("targetNodes", List.of(0, 42, 1)) - .yields(); - - assertError(query, "Target nodes do not exist in the in-memory graph: ['42']"); - } } diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumerTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumerTest.java index b76ce8c0511..aa9543c0d84 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumerTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamComputationResultConsumerTest.java @@ -53,7 +53,6 @@ class DfsStreamComputationResultConsumerTest { @Test void shouldNotComputePath() { - when(graphMock.isEmpty()).thenReturn(false); when(graphMock.toOriginalNodeId(anyLong())).then(returnsFirstArg()); when(computationResultMock.graph()).thenReturn(graphMock); @@ -82,7 +81,6 @@ void shouldNotComputePath() { @Test void shouldComputePath() { - when(graphMock.isEmpty()).thenReturn(false); when(graphMock.toOriginalNodeId(anyLong())).then(returnsFirstArg()); when(computationResultMock.graph()).thenReturn(graphMock); diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamProcTest.java index 20340817925..605d3d5e24a 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamProcTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/DfsStreamProcTest.java @@ -228,28 +228,4 @@ static Stream pathQueryBuilders() { ); } - @Test - void failOnInvalidSourceNode() { - loadCompleteGraph(DEFAULT_GRAPH_NAME); - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("dfs") - .streamMode() - .addParameter("sourceNode", 42) - .yields(); - - assertError(query, "Source node does not exist in the in-memory graph: `42`"); - } - - @Test - void failOnInvalidEndNode() { - loadCompleteGraph(DEFAULT_GRAPH_NAME); - String query = GdsCypher.call(DEFAULT_GRAPH_NAME) - .algo("dfs") - .streamMode() - .addParameter("sourceNode", 0) - .addParameter("targetNodes", Arrays.asList(0, 42, 1)) - .yields(); - - assertError(query, "Target nodes do not exist in the in-memory graph: ['42']"); - } } diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumerTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumerTest.java index df1ff58f20c..eded824696b 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumerTest.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/traverse/TraverseStreamComputationResultConsumerTest.java @@ -43,7 +43,6 @@ void shouldNotComputePath() { 0L, HugeLongArray.of(1L, 2L), l -> l, - false, TestResult::new, false, pathFactoryFacadeMock, @@ -70,7 +69,6 @@ void shouldComputePath() { 0L, HugeLongArray.of(1L, 2L), l -> l, - false, TestResult::new, true, pathFactoryFacadeMock, diff --git a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelStreamComputationResultConsumer.java b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelStreamComputationResultConsumer.java index 32c9d2a364d..628d17e6689 100644 --- a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelStreamComputationResultConsumer.java +++ b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelStreamComputationResultConsumer.java @@ -49,46 +49,46 @@ public Stream consume( return runWithExceptionLogging( "Result streaming failed", executionContext.log(), - () -> { - var result = computationResult.result(); - if (result.isEmpty()) { - return Stream.empty(); - } - var nodeValues = result.get().nodeValues(); - // TODO: reduce the inner iteration to what's visible upfront - // TODO: map the element to a property lookup function upfront - // for every node - return LongStream.range(IdMap.START_NODE_ID, computationResult.graph().nodeCount()).mapToObj(nodeId -> { - // for every schema element - Map values = nodeValues.schema().elements() - .stream() - // if its visible - .filter(element -> element.visibility() == PregelSchema.Visibility.PUBLIC) - // collect a String->Object map - .collect(Collectors.toMap( - // of the element's property key - Element::propertyKey, - // to a value - element -> { - // retrieved based on the elements property type - switch (element.propertyType()) { - case LONG: - return nodeValues.longProperties(element.propertyKey()).get(nodeId); - case DOUBLE: - return nodeValues.doubleProperties(element.propertyKey()).get(nodeId); - case DOUBLE_ARRAY: - return nodeValues.doubleArrayProperties(element.propertyKey()).get(nodeId); - case LONG_ARRAY: - return nodeValues.longArrayProperties(element.propertyKey()).get(nodeId); - default: - throw new IllegalArgumentException("Unsupported property type: " + element.propertyType()); - } - } - )); - return new PregelStreamResult(computationResult.graph().toOriginalNodeId(nodeId), values); - }); + () -> computationResult.result() + .map(result -> { + var nodeValues = result.nodeValues(); + // TODO: reduce the inner iteration to what's visible upfront + // TODO: map the element to a property lookup function upfront + // for every node + return LongStream.range(IdMap.START_NODE_ID, computationResult.graph().nodeCount()) + .mapToObj(nodeId -> { + // for every schema element + Map values = nodeValues.schema().elements() + .stream() + // if its visible + .filter(element -> element.visibility() == PregelSchema.Visibility.PUBLIC) + // collect a String->Object map + .collect(Collectors.toMap( + // of the element's property key + Element::propertyKey, + // to a value + element -> { + // retrieved based on the elements property type + switch (element.propertyType()) { + case LONG: + return nodeValues.longProperties(element.propertyKey()).get(nodeId); + case DOUBLE: + return nodeValues.doubleProperties(element.propertyKey()).get(nodeId); + case DOUBLE_ARRAY: + return nodeValues.doubleArrayProperties(element.propertyKey()).get( + nodeId); + case LONG_ARRAY: + return nodeValues.longArrayProperties(element.propertyKey()) + .get(nodeId); + default: + throw new IllegalArgumentException("Unsupported property type: " + element.propertyType()); + } + } + )); + return new PregelStreamResult(computationResult.graph().toOriginalNodeId(nodeId), values); + }); - } + }).orElseGet(Stream::empty) ); } } diff --git a/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelProcTest.java b/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelProcTest.java index 914cca1a466..845d8e023d4 100644 --- a/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelProcTest.java +++ b/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelProcTest.java @@ -41,7 +41,7 @@ import org.neo4j.gds.beta.pregel.PregelSchema; import org.neo4j.gds.beta.pregel.context.ComputeContext; import org.neo4j.gds.catalog.GraphProjectProc; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; @@ -464,7 +464,7 @@ public void compute(ComputeContext context, Messages mess context.setNodeValue(LONG_ARRAY_KEY, new long[]{1, 3, 3, 7}); context.setNodeValue(DOUBLE_ARRAY_KEY, new double[]{1, 9, 8, 4}); } - }, Pools.DEFAULT, progressTracker); + }, DefaultPool.INSTANCE, progressTracker); } @Override diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnStreamSpecification.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnStreamSpecification.java index 4290cc96a45..b9212aa4866 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnStreamSpecification.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/filteredknn/FilteredKnnStreamSpecification.java @@ -29,6 +29,8 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; + @GdsCallable(name = "gds.alpha.knn.filtered.stream", executionMode = ExecutionMode.STREAM) public class FilteredKnnStreamSpecification implements AlgorithmSpec, FilteredKnnFactory> { @@ -49,8 +51,10 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - return computationResult.result() + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() .map(result -> { var graph = computationResult.graph(); return result.similarityResultStream() @@ -60,7 +64,7 @@ public ComputationResultConsumer newConfigFunction() @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - var graph = computationResult.graph(); - var similarityResultStream = computationResult.result() - .map(NodeSimilarityResult::streamResult) - .orElseGet(Stream::empty); + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() + .map(result -> { + var graph = computationResult.graph(); + var similarityResultStream = result.streamResult(); + return similarityResultStream.map(internalSimilarityResult -> { + internalSimilarityResult.node1 = graph.toOriginalNodeId(internalSimilarityResult.node1); + internalSimilarityResult.node2 = graph.toOriginalNodeId(internalSimilarityResult.node2); - return similarityResultStream.map(internalSimilarityResult -> { - internalSimilarityResult.node1 = graph.toOriginalNodeId(internalSimilarityResult.node1); - internalSimilarityResult.node2 = graph.toOriginalNodeId(internalSimilarityResult.node2); - - return internalSimilarityResult; - }); - }; + return internalSimilarityResult; + }); + }).orElseGet(Stream::empty) + ); } } diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnStreamSpecification.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnStreamSpecification.java index 46fb11d55fd..e7d8a231402 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnStreamSpecification.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnStreamSpecification.java @@ -29,6 +29,7 @@ import java.util.stream.Stream; +import static org.neo4j.gds.LoggingUtil.runWithExceptionLogging; import static org.neo4j.gds.similarity.knn.KnnProc.KNN_DESCRIPTION; @GdsCallable(name = "gds.knn.stream", description = KNN_DESCRIPTION, executionMode = ExecutionMode.STREAM) @@ -51,8 +52,10 @@ public NewConfigFunction newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - return computationResult.result() + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() .map(result -> { var graph = computationResult.graph(); return result.streamSimilarityResult() @@ -62,7 +65,7 @@ public ComputationResultConsumer newConfigFunction() { @Override public ComputationResultConsumer> computationResultConsumer() { - return (computationResult, executionContext) -> { - return computationResult.result() + return (computationResult, executionContext) -> runWithExceptionLogging( + "Result streaming failed", + executionContext.log(), + () -> computationResult.result() .map(result -> { var graph = computationResult.graph(); @@ -62,7 +65,7 @@ public ComputationResultConsumer. + */ +package org.neo4j.gds.similarity.nodesim; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.extension.Neo4jGraph; + +import static org.assertj.core.api.Assertions.assertThatNoException; + +class NodeSimilarityStreamProcCleanTest extends BaseProcTest { + + @Neo4jGraph + public static final String DB_CYPHER = + "CREATE" + + " (a:Person {id: 0, name: 'Alice'})" + + ", (b:Person {id: 1, name: 'Bob'})" + + ", (c:Person {id: 2, name: 'Charlie'})" + + ", (d:Person {id: 3, name: 'Dave'})" + + ", (i1:Item {id: 10, name: 'p1'})" + + ", (i2:Item {id: 11, name: 'p2'})" + + ", (i3:Item {id: 12, name: 'p3'})" + + ", (i4:Item {id: 13, name: 'p4'})" + + ", (a)-[:LIKES]->(i1)" + + ", (a)-[:LIKES]->(i2)" + + ", (a)-[:LIKES]->(i3)" + + ", (b)-[:LIKES]->(i1)" + + ", (b)-[:LIKES]->(i2)" + + ", (c)-[:LIKES]->(i3)"; + + + @BeforeEach + void setUp() throws Exception { + registerProcedures( + NodeSimilarityStreamProc.class, + GraphProjectProc.class + ); + + runQuery("CALL gds.graph.project('g', '*', 'LIKES')"); + } + + @AfterEach + void tearDown() { + GraphStoreCatalog.removeAllLoadedGraphs(); + } + + @Test + void shouldWorkWithCosineSimilarityMetric() { + assertThatNoException().isThrownBy( + () -> runQuery("CALL gds.nodeSimilarity.stream('g', {similarityCutoff: 0.0, similarityMetric: 'cosine'})") + ); + } +} diff --git a/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java b/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java index 096db14b00d..67e57de64ac 100644 --- a/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java +++ b/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java @@ -167,7 +167,6 @@ private static void features(Stream.Builder builder) { )) .add(value("featurePartitionedScan", GdsFeatureToggles.USE_PARTITIONED_SCAN.isEnabled())) .add(value("featureBitIdMap", GdsFeatureToggles.USE_BIT_ID_MAP.isEnabled())) - .add(value("featureShardedIdMap", GdsFeatureToggles.USE_SHARDED_ID_MAP.isEnabled())) .add(value( "featureUncompressedAdjacencyList", GdsFeatureToggles.USE_UNCOMPRESSED_ADJACENCY_LIST.isEnabled() diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java index 93521804ba4..a4bce7fd3db 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java @@ -106,7 +106,7 @@ void loadFromPropertiesRequiresVersion() { } private Optional findVersion(Path file) throws IOException { - Pattern pattern = Pattern.compile(".*gdsVersion = '(\\d\\.\\d\\.\\d+(-alpha\\d+|-beta\\d+)?)'.*"); + Pattern pattern = Pattern.compile(".*gdsBaseVersion = '(\\d\\.\\d\\.\\d+(-alpha\\d+|-beta\\d+)?)'.*"); return Files.lines(file, StandardCharsets.UTF_8) .flatMap(line -> { var matcher = pattern.matcher(line); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 0fd69cf1765..41497499828 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -92,6 +92,31 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.8", "Neo4j Settings 5.8 (placeholder)", + "Neo4j 5.9", + "Neo4j 5.9 (placeholder)", + "Neo4j Settings 5.9", + "Neo4j Settings 5.9 (placeholder)", + + "Neo4j 5.10", + "Neo4j 5.10 (placeholder)", + "Neo4j Settings 5.10", + "Neo4j Settings 5.10 (placeholder)", + + "Neo4j 5.11", + "Neo4j 5.11 (placeholder)", + "Neo4j Settings 5.11", + "Neo4j Settings 5.11 (placeholder)", + + "Neo4j 5.12", + "Neo4j 5.12 (placeholder)", + "Neo4j Settings 5.12", + "Neo4j Settings 5.12 (placeholder)", + + "Neo4j 5.13", + "Neo4j 5.13 (placeholder)", + "Neo4j Settings 5.13", + "Neo4j Settings 5.13 (placeholder)", + "Neo4j DEV", "Neo4j DEV (placeholder)", "Neo4j Settings DEV", @@ -216,14 +241,52 @@ void testSysInfoProc() throws IOException { "Neo4j 5.8" ); break; - case V_Dev: + case V_5_9: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.9 (placeholder)", + "Neo4j Settings 5.9", + "Neo4j 5.9 (placeholder)", + "Neo4j 5.9" + ); + break; + case V_5_10: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.10 (placeholder)", + "Neo4j Settings 5.10", + "Neo4j 5.10 (placeholder)", + "Neo4j 5.10" + ); + break; + case V_5_11: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.11 (placeholder)", + "Neo4j Settings 5.11", + "Neo4j 5.11 (placeholder)", + "Neo4j 5.11" + ); + break; + case V_5_12: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.12 (placeholder)", + "Neo4j Settings 5.12", + "Neo4j 5.12 (placeholder)", + "Neo4j 5.12" + ); + break; + case V_5_13: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.13 (placeholder)", + "Neo4j Settings 5.13", + "Neo4j 5.13 (placeholder)", + "Neo4j 5.13" + ); + break; + case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC", - "Neo4j Settings DEV (placeholder)", - "Neo4j Settings DEV", + "Neo4j Settings RC (placeholder)", "Neo4j RC", - "Neo4j DEV (placeholder)", - "Neo4j DEV" + "Neo4j RC (placeholder)" ); break; default: @@ -300,7 +363,6 @@ void testSysInfoProc() throws IOException { .containsEntry("featureParallelPropertyValueIndex", GdsFeatureToggles.USE_PARALLEL_PROPERTY_VALUE_INDEX.isEnabled()) .containsEntry("featurePartitionedScan", GdsFeatureToggles.USE_PARTITIONED_SCAN.isEnabled()) .containsEntry("featureBitIdMap", GdsFeatureToggles.USE_BIT_ID_MAP.isEnabled()) - .containsEntry("featureShardedIdMap", GdsFeatureToggles.USE_SHARDED_ID_MAP.isEnabled()) .containsEntry("featureUncompressedAdjacencyList", GdsFeatureToggles.USE_UNCOMPRESSED_ADJACENCY_LIST.isEnabled()) .containsEntry("featurePackedAdjacencyList", GdsFeatureToggles.USE_PACKED_ADJACENCY_LIST.isEnabled()) .containsEntry("featureReorderedAdjacencyList", GdsFeatureToggles.USE_REORDERED_ADJACENCY_LIST.isEnabled()) diff --git a/procedure-collector/build.gradle b/procedure-collector/build.gradle index 6e3d84126c5..b65aa3860d8 100644 --- a/procedure-collector/build.gradle +++ b/procedure-collector/build.gradle @@ -10,7 +10,7 @@ dependencies { annotationProcessor group: 'org.immutables', name: 'builder', version: ver.'immutables' annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' - implementation group: 'com.google.auto.service', name: 'auto-service', version: ver.'auto-service' + compileOnly group: 'com.google.auto.service', name: 'auto-service', version: ver.'auto-service' implementation project(':annotations') implementation project(':executor') diff --git a/progress-tracking/src/main/java/org/neo4j/gds/core/utils/progress/BatchingProgressLogger.java b/progress-tracking/src/main/java/org/neo4j/gds/core/utils/progress/BatchingProgressLogger.java index ef70ed9f6f0..005e0c70b35 100644 --- a/progress-tracking/src/main/java/org/neo4j/gds/core/utils/progress/BatchingProgressLogger.java +++ b/progress-tracking/src/main/java/org/neo4j/gds/core/utils/progress/BatchingProgressLogger.java @@ -124,7 +124,7 @@ private synchronized void doLogPercentage(Supplier msgFactory, long prog String message = msgFactory != NO_MESSAGE ? msgFactory.get() : null; progressCounter.add(progress); int nextPercentage = (int) ((progressCounter.sum() / (double) taskVolume) * 100); - if (globalPercentage < nextPercentage) { + if (globalPercentage < nextPercentage && globalPercentage < 100) { globalPercentage = nextPercentage; if (message == null || message.isEmpty()) { logProgress(nextPercentage); diff --git a/progress-tracking/src/test/java/org/neo4j/gds/core/utils/BatchingProgressLoggerTest.java b/progress-tracking/src/test/java/org/neo4j/gds/core/utils/BatchingProgressLoggerTest.java index 4fc44bf7ab0..027ed7df697 100644 --- a/progress-tracking/src/test/java/org/neo4j/gds/core/utils/BatchingProgressLoggerTest.java +++ b/progress-tracking/src/test/java/org/neo4j/gds/core/utils/BatchingProgressLoggerTest.java @@ -182,6 +182,18 @@ void shouldLog100OnlyOnce() { .containsExactly("Test 100%"); } + @Test + void shouldNotExceed100Percent() { + TestLog log = Neo4jProxy.testLog(); + var testProgressLogger = new BatchingProgressLogger(log, Tasks.leaf("Test"), 1); + testProgressLogger.reset(1); + testProgressLogger.logProgress(1); // reaches 100 % + testProgressLogger.logProgress(1); // exceeds 100 % + assertThat(log.getMessages(TestLog.INFO)) + .extracting(Extractors.removingThreadId()) + .containsExactly("Test 100%"); + } + @Test void closesThreadLocal() { var logger = new BatchingProgressLogger( diff --git a/settings.gradle b/settings.gradle index 9379ed4f1cb..cf9d1efecf8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -154,6 +154,18 @@ project(':neo4j-kernel-adapter-5.7').projectDir = file('compatibility/5.7/neo4j- include('neo4j-kernel-adapter-5.8') project(':neo4j-kernel-adapter-5.8').projectDir = file('compatibility/5.8/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.9') +project(':neo4j-kernel-adapter-5.9').projectDir = file('compatibility/5.9/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.10') +project(':neo4j-kernel-adapter-5.10').projectDir = file('compatibility/5.10/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.11') +project(':neo4j-kernel-adapter-5.11').projectDir = file('compatibility/5.11/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.12') +project(':neo4j-kernel-adapter-5.12').projectDir = file('compatibility/5.12/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -256,6 +268,18 @@ project(':storage-engine-adapter-5.7').projectDir = file('compatibility/5.7/stor include('storage-engine-adapter-5.8') project(':storage-engine-adapter-5.8').projectDir = file('compatibility/5.8/storage-engine-adapter') +include('storage-engine-adapter-5.9') +project(':storage-engine-adapter-5.9').projectDir = file('compatibility/5.9/storage-engine-adapter') + +include('storage-engine-adapter-5.10') +project(':storage-engine-adapter-5.10').projectDir = file('compatibility/5.10/storage-engine-adapter') + +include('storage-engine-adapter-5.11') +project(':storage-engine-adapter-5.11').projectDir = file('compatibility/5.11/storage-engine-adapter') + +include('storage-engine-adapter-5.12') +project(':storage-engine-adapter-5.12').projectDir = file('compatibility/5.12/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java index 0dafb6afcbd..726ac5a9c77 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java @@ -25,7 +25,6 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; @@ -186,9 +185,6 @@ public static MutableGraphSchema filterSchema(GraphSchema inputGraphSchema, Node .relationshipSchema() .filter(filteredRelationshipTypes) ); - if (relationshipSchema.availableTypes().isEmpty()) { - relationshipSchema.addRelationshipType(RelationshipType.ALL_RELATIONSHIPS, Direction.DIRECTED); - } return MutableGraphSchema.of(nodeSchema, relationshipSchema, Map.of()); } diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java index 53de58f7b10..cfdfbbaa818 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java @@ -26,6 +26,7 @@ import org.neo4j.gds.beta.filter.expression.EvaluationContext; import org.neo4j.gds.beta.filter.expression.Expression; import org.neo4j.gds.core.Aggregation; +import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -42,6 +43,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.LongStream; import static org.neo4j.gds.api.AdjacencyCursor.NOT_FOUND; @@ -106,11 +108,13 @@ private static SingleTypeRelationships filterRelationshipType( var propertyConfigs = propertyKeys .stream() - .map(propertyKey -> GraphFactory.PropertyConfig.of( - propertyKey, - Aggregation.NONE, - graphStore.relationshipPropertyValues(relType, propertyKey).defaultValue() - )) + .map( + propertyKey -> GraphFactory.PropertyConfig.of( + propertyKey, + Aggregation.NONE, + graphStore.relationshipPropertyValues(relType, propertyKey).defaultValue() + ) + ) .collect(Collectors.toList()); var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() @@ -128,13 +132,23 @@ private static SingleTypeRelationships filterRelationshipType( .boxed() .collect(Collectors.toMap(propertyKeys::get, Function.identity())); + // computing the count is expected to be a lot cheaper than bad partitioning + var relevantRelationshipsCount = concurrency > 1 + ? ParallelUtil.parallelStream( + LongStream.range(0, outputNodes.nodeCount()), + concurrency, + stream -> stream.map( + id -> compositeIterator.degree(inputNodes.toMappedNodeId(outputNodes.toOriginalNodeId(id))) + ).sum() + ) + : graphStore.relationshipCount(relType); + var relationshipFilterTasks = PartitionUtils.degreePartition( outputNodes.nodeCount(), - graphStore.relationshipCount(relType), - compositeIterator::degree, + relevantRelationshipsCount, + nodeId -> compositeIterator.degree(inputNodes.toMappedNodeId(outputNodes.toOriginalNodeId(nodeId))), concurrency, - partition -> - new RelationshipFilterTask( + partition -> new RelationshipFilterTask( partition, relationshipExpr, compositeIterator.concurrentCopy(), @@ -155,7 +169,7 @@ private static SingleTypeRelationships filterRelationshipType( .executor(executorService) .run(); - return relationshipsBuilder.build(); + return relationshipsBuilder.build(); } private RelationshipsFilter() {} diff --git a/subgraph-filtering/src/test/java/org/neo4j/gds/beta/filter/NodesFilterTest.java b/subgraph-filtering/src/test/java/org/neo4j/gds/beta/filter/NodesFilterTest.java index ce8a9d31cbd..e269fb13bbc 100644 --- a/subgraph-filtering/src/test/java/org/neo4j/gds/beta/filter/NodesFilterTest.java +++ b/subgraph-filtering/src/test/java/org/neo4j/gds/beta/filter/NodesFilterTest.java @@ -23,7 +23,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.beta.filter.expression.ExpressionParser; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.ExecutorServiceUtil; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -60,7 +60,7 @@ void basicFiltering() throws ParseException { ExpressionParser.parse("n:A AND n.p > 1.0", Map.of()), 1, Map.of(), - Pools.DEFAULT_SINGLE_THREAD_POOL, + ExecutorServiceUtil.DEFAULT_SINGLE_THREAD_POOL, ProgressTracker.NULL_TRACKER ); diff --git a/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java b/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java index a9484fc9801..b1e151f913e 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java +++ b/test-utils/src/main/java/org/neo4j/gds/GraphLoaderBuilders.java @@ -31,7 +31,7 @@ import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphLoader; import org.neo4j.gds.core.ImmutableGraphLoader; -import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.JobId; @@ -194,7 +194,7 @@ public static GraphLoader createGraphLoader( .databaseId(DatabaseId.of(databaseService)) .dependencyResolver(GraphDatabaseApiProxy.dependencyResolver(databaseService)) .transactionContext(transactionContext.orElseGet(() -> TestSupport.fullAccessTransaction(databaseService))) - .executor(executorService.orElse(Pools.DEFAULT)) + .executor(executorService.orElse(DefaultPool.INSTANCE)) .terminationFlag(terminationFlag.orElse(TerminationFlag.RUNNING_TRUE)) .taskRegistryFactory(EmptyTaskRegistryFactory.INSTANCE) .userLogRegistryFactory(EmptyUserLogRegistryFactory.INSTANCE) diff --git a/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java b/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java index dfaa7c26aea..b80df0e0095 100644 --- a/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java +++ b/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java @@ -35,9 +35,11 @@ import java.util.function.Function; import static java.util.Collections.emptyMap; +import static org.junit.jupiter.api.Assertions.fail; import static org.neo4j.gds.compat.GraphDatabaseApiProxy.applyInTransaction; import static org.neo4j.gds.compat.GraphDatabaseApiProxy.runInTransaction; import static org.neo4j.gds.compat.GraphDatabaseApiProxy.runQueryWithoutClosingTheResult; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; import static org.neo4j.internal.kernel.api.security.AccessMode.Static.READ; public final class QueryRunner { @@ -203,6 +205,15 @@ public static void runQueryWithResultConsumer( }); } + public static void runFailingQuery(GraphDatabaseService db, String query, Map queryParameters, Consumer exceptionConsumer) { + try { + QueryRunner.runQueryWithResultConsumer(db, query, queryParameters, Result::resultAsString); + fail(formatWithLocale("Expected an exception to be thrown by query:\n%s", query)); + } catch (Throwable e) { + exceptionConsumer.accept(e); + } + } + private static KernelTransaction.Revertable withUsername(Transaction tx, String username, String databaseName) { InternalTransaction topLevelTransaction = (InternalTransaction) tx; AuthSubject subject = topLevelTransaction.securityContext().subject(); diff --git a/test-utils/src/main/java/org/neo4j/gds/TestTaskStore.java b/test-utils/src/main/java/org/neo4j/gds/TestTaskStore.java index 339f7f8e84a..30d22215116 100644 --- a/test-utils/src/main/java/org/neo4j/gds/TestTaskStore.java +++ b/test-utils/src/main/java/org/neo4j/gds/TestTaskStore.java @@ -43,6 +43,7 @@ public void store(String username, JobId jobId, Task task) { @Override public void remove(String username, JobId jobId) { + super.remove(username, jobId); tasks.remove(jobId); } diff --git a/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java b/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java index 6846829f57a..173dc7ffaa1 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/huge/DirectIdMap.java @@ -39,6 +39,11 @@ public DirectIdMap(long nodeCount) { this.nodeCount = nodeCount; } + @Override + public String typeId() { + return NO_TYPE; + } + @Override public long toMappedNodeId(long originalNodeId) { return originalNodeId; diff --git a/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java b/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java index 412a52a9d34..e0c114266f2 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/idmap/IdMapBuilderTest.java @@ -28,8 +28,8 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.core.concurrency.DefaultPool; import org.neo4j.gds.core.concurrency.ParallelUtil; -import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.utils.partition.PartitionUtils; @@ -41,7 +41,7 @@ import java.util.Optional; import java.util.Random; import java.util.SplittableRandom; -import java.util.function.LongFunction; +import java.util.function.IntFunction; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -111,7 +111,7 @@ void testContains( @ForAll long seed ) { var originalIds = generateShuffledIds(idOffset, nodeCount, seed); - var idMapAndHighestId = buildFrom(originalIds, concurrency); + var idMapAndHighestId = buildFrom(originalIds.clone(), concurrency); var idMap = idMapAndHighestId.idMap(); var highestOriginalId = idMapAndHighestId.highestOriginalId(); @@ -152,7 +152,7 @@ void testToMappedNodeId( @ForAll long seed ) { var originalIds = generateShuffledIds(idOffset, nodeCount, seed); - var idMap = buildIdMapFrom(originalIds, concurrency); + var idMap = buildIdMapFrom(originalIds.clone(), concurrency); var mappedNodeIds = new long[originalIds.length]; var actualNodeCount = idMap.nodeCount(); @@ -202,7 +202,7 @@ void testToOriginalNodeId( @ForAll long seed ) { var originalIds = generateShuffledIds(idOffset, nodeCount, seed); - var idMap = buildIdMapFrom(originalIds, concurrency); + var idMap = buildIdMapFrom(originalIds.clone(), concurrency); var actualOriginalIds = new long[originalIds.length]; Arrays.fill(actualOriginalIds, -1); @@ -241,7 +241,7 @@ void testBuildParallel( } }, Optional.empty()); - ParallelUtil.run(tasks, Pools.DEFAULT); + ParallelUtil.run(tasks, DefaultPool.INSTANCE); var idMap = idMapBuilder.build(LabelInformationBuilders.allNodes(), highestOriginalId, concurrency); @@ -271,12 +271,12 @@ void testLabels( var expectedLabels = new HashMap>(); var noLabel = List.of(NodeLabel.ALL_NODES); - var idMap = buildFromWithLabels(originalIds, concurrency, originalId -> { + var idMap = buildFromWithLabels(originalIds.clone(), concurrency, index -> { var labelCount = rng.nextInt(allLabels.length); var labels = labelCount > 0 ? Arrays.stream(allLabels).limit(labelCount).collect(Collectors.toList()) : noLabel; - expectedLabels.put(originalId, labels); + expectedLabels.put(originalIds[index], labels); return labels; }).idMap(); @@ -308,7 +308,7 @@ private IdMapAndHighestId buildFrom(long[] originalIds, int concurrency) { private IdMapAndHighestId buildFromWithLabels( long[] originalIds, int concurrency, - LongFunction> labelFn + IntFunction> labelFn ) { return buildFromWithLabels(originalIds, concurrency, Optional.ofNullable(labelFn)); } @@ -316,7 +316,7 @@ private IdMapAndHighestId buildFromWithLabels( private IdMapAndHighestId buildFromWithLabels( long[] originalIds, int concurrency, - Optional>> labelFn + Optional>> labelFn ) { // number of ids we want to insert at once int batchLength = originalIds.length; @@ -330,8 +330,9 @@ private IdMapAndHighestId buildFromWithLabels( var labelInformationBuilder = labelFn.map(lFn -> { var multiLabelBuilder = LabelInformationBuilders.multiLabelWithCapacity(originalIds.length); - for (long originalNodeId : originalIds) { - lFn.apply(originalNodeId).forEach(label -> multiLabelBuilder.addNodeIdToLabel(label, originalNodeId)); + for (int i = 0; i < originalIds.length; i++) { + long originalId = originalIds[i]; + lFn.apply(i).forEach(label -> multiLabelBuilder.addNodeIdToLabel(label, originalId)); } return multiLabelBuilder; }).orElseGet(LabelInformationBuilders::allNodes); diff --git a/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java b/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java index d62278f2790..17da4853bd2 100644 --- a/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java +++ b/test-utils/src/main/java/org/neo4j/gds/core/loading/TestIdMap.java @@ -50,6 +50,11 @@ private TestIdMap( this.highestOriginalId = highestOriginalId; } + @Override + public String typeId() { + return NO_TYPE; + } + @Override public long toOriginalNodeId(long mappedNodeId) { return this.reverseMap.getOrDefault(mappedNodeId, NOT_FOUND); diff --git a/test-utils/src/main/java/org/neo4j/gds/extension/FakeClockSupportExtension.java b/test-utils/src/main/java/org/neo4j/gds/extension/FakeClockSupportExtension.java index 44bbe1e5bad..237a72a6aa2 100644 --- a/test-utils/src/main/java/org/neo4j/gds/extension/FakeClockSupportExtension.java +++ b/test-utils/src/main/java/org/neo4j/gds/extension/FakeClockSupportExtension.java @@ -47,7 +47,9 @@ public void beforeEach(ExtensionContext context) throws Exception { @Override public void afterEach(ExtensionContext context) throws Exception { - ClockService.setClock(clockBefore); - clockBefore = null; + if (clockBefore != null) { + ClockService.setClock(clockBefore); + clockBefore = null; + } } } diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index c90fb85e49b..430de232265 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -78,6 +78,7 @@ public final class GdlFactory extends CSRGraphStoreFactory graphName, Optional graphProjectConfig, Optional nodeIdFunction, - Optional graphCapabilities + Optional graphCapabilities, + Optional idMapBuilderType ) { var config = graphProjectConfig.orElseGet(() -> ImmutableGraphProjectFromGdlConfig.builder() .username(userName.orElse(Username.EMPTY_USERNAME.username())) @@ -123,7 +125,8 @@ static GdlFactory gdlFactory( config, graphDimensions, databaseId.orElse(GdlSupportPerMethodExtension.DATABASE_ID), - capabilities + capabilities, + idMapBuilderType.orElse(IdMap.NO_TYPE) ); } @@ -132,7 +135,8 @@ private GdlFactory( GraphProjectFromGdlConfig graphProjectConfig, GraphDimensions graphDimensions, DatabaseId databaseId, - Capabilities capabilities + Capabilities capabilities, + String idMapBuilderType ) { super( graphProjectConfig, @@ -142,6 +146,7 @@ private GdlFactory( ); this.gdlHandler = gdlHandler; this.databaseId = databaseId; + this.idMapBuilderType = idMapBuilderType; } public long nodeId(String variable) { @@ -181,10 +186,13 @@ public CSRGraphStore build() { private Nodes loadNodes() { var nodesBuilder = GraphFactory.initNodesBuilder() .maxOriginalId(dimensions.highestPossibleNodeCount() - 1) + .nodeCount(gdlHandler.getVertices().size()) .hasLabelInformation(true) .hasProperties(true) + .deduplicateIds(false) .concurrency(1) .propertyState(graphProjectConfig.propertyState()) + .idMapBuilderType(this.idMapBuilderType) .build(); gdlHandler.getVertices().forEach(vertex -> { @@ -382,7 +390,7 @@ static GraphDimensions of(GDLHandler gdlHandler) { .stream() .map(Vertex::getId) .max(Long::compareTo) - .orElse((long) nodeCount); + .orElse((long) nodeCount - 1); var relCount = gdlHandler.getEdges().size(); var nodePropertyDimensions = new HashMap>();