From e5d5b13e7991137cef4d8bb5ecff14599692726f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 13 Jan 2023 11:00:16 +0100 Subject: [PATCH 001/400] Remove dev compat layer and training material These only live on master --- .../main/java/org/neo4j/gds/compat/ProxyUtil.java | 1 - .../java/org/neo4j/gds/compat/Neo4jVersion.java | 13 ++----------- .../java/org/neo4j/gds/compat/Neo4jVersionTest.java | 2 +- .../test/java/org/neo4j/gds/SysInfoProcTest.java | 10 ---------- 4 files changed, 3 insertions(+), 23 deletions(-) diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java index 6d9a8ab2191..e74f381612d 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/ProxyUtil.java @@ -191,7 +191,6 @@ private static Neo4jVersionInfo loadNeo4jVersion() { .reason(e) .build() ) - .neo4jVersion(Neo4jVersion.V_Dev) .build(); } } diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 3035e9c6325..196787de6ed 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -30,8 +30,7 @@ public enum Neo4jVersion { V_4_4, V_5_1, V_5_2, - V_5_3, - V_Dev; + V_5_3; @Override public String toString() { @@ -44,19 +43,13 @@ public String toString() { return "5.2"; case V_5_3: return "5.3"; - case V_Dev: - return "dev"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } } public MajorMinorVersion semanticVersion() { - if (this == V_Dev) { - return ImmutableMajorMinorVersion.of(5, 4); - } - - String version = toString(); + String version = toString(); var subVersions = version.split("\\."); if (subVersions.length < 2) { @@ -122,8 +115,6 @@ static Neo4jVersion parse(String version) { return Neo4jVersion.V_5_2; } else if (minorVersion == 3) { return Neo4jVersion.V_5_3; - } else if (minorVersion > 3) { - return Neo4jVersion.V_Dev; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index d07bb441c3c..b7bb4fcdd2a 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -40,7 +40,6 @@ class Neo4jVersionTest { "4.4.14, V_4_4", "5.1.0, V_5_1", "5.2.0, V_5_2", - "5.4.0, V_Dev", "5.1.0-dev, V_5_1", "5.2.0-dev, V_5_2", "5.2.0, V_5_2", @@ -54,6 +53,7 @@ void testParse(String input, Neo4jVersion expected) { @CsvSource({ "dev", "4.3", // EOL + "5.4.0", // was dev at cutoff point "5.dev", "dev.5", "5.0", // 5.0 was never released to the public diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index e6c11a96723..d04f83f734e 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -150,16 +150,6 @@ void testSysInfoProc() throws IOException { "Neo4j 5.3" ); break; - case V_Dev: - expectedCompatibilities = Set.of( - "Neo4j Settings RC", - "Neo4j Settings DEV (placeholder)", - "Neo4j Settings DEV", - "Neo4j RC", - "Neo4j DEV (placeholder)", - "Neo4j DEV" - ); - break; default: throw new IllegalStateException("Unexpected Neo4j version: " + neo4jVersion); } From 11b3bdd6a231d701dd8e0bf703cae2204a6e159c Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 13 Jan 2023 10:15:40 +0000 Subject: [PATCH 002/400] Align GDS on cluster docs with Neo4j 5.x --- doc/modules/ROOT/content-nav.adoc | 4 +- .../ROOT/pages/installation/index.adoc | 2 +- .../installation-causal-cluster.adoc | 11 ---- .../installation-neo4j-cluster.adoc | 11 ++++ .../ROOT/pages/installation/neo4j-server.adoc | 4 +- doc/modules/ROOT/pages/introduction.adoc | 2 +- .../pages/production-deployment/index.adoc | 2 +- ...causal-cluster.adoc => neo4j-cluster.adoc} | 59 ++++++++++--------- 8 files changed, 48 insertions(+), 47 deletions(-) delete mode 100644 doc/modules/ROOT/pages/installation/installation-causal-cluster.adoc create mode 100644 doc/modules/ROOT/pages/installation/installation-neo4j-cluster.adoc rename doc/modules/ROOT/pages/production-deployment/{causal-cluster.adoc => neo4j-cluster.adoc} (91%) diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index cc705b62235..bb25da160f4 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -6,7 +6,7 @@ ** xref:installation/neo4j-server.adoc[] ** xref:installation/installation-enterprise-edition.adoc[] ** xref:installation/installation-docker.adoc[] -** xref:installation/installation-causal-cluster.adoc[] +** xref:installation/installation-neo4j-cluster.adoc[] ** xref:installation/installation-apache-arrow.adoc[] ** xref:installation/additional-config-parameters.adoc[] ** xref:installation/System-requirements.adoc[] @@ -147,7 +147,7 @@ * xref:production-deployment/index.adoc[] ** xref:production-deployment/transaction-handling.adoc[] ** xref:production-deployment/fabric.adoc[] -** xref:production-deployment/causal-cluster.adoc[] +** xref:production-deployment/neo4j-cluster.adoc[] ** xref:production-deployment/feature-toggles.adoc[] * xref:python-client/index.adoc[] * Appendix diff --git a/doc/modules/ROOT/pages/installation/index.adoc b/doc/modules/ROOT/pages/installation/index.adoc index cc36aa9364a..e3cbbb664cd 100644 --- a/doc/modules/ROOT/pages/installation/index.adoc +++ b/doc/modules/ROOT/pages/installation/index.adoc @@ -15,7 +15,7 @@ This chapter is divided into the following sections: . xref:installation/neo4j-server.adoc[Neo4j Server] . xref:installation/installation-enterprise-edition.adoc[Enterprise Edition Configuration] . xref:installation/installation-docker.adoc[Neo4j Docker] -. xref:installation/installation-causal-cluster.adoc[Neo4j Causal Cluster] +. xref:installation/installation-neo4j-cluster.adoc[Neo4j cluster] . xref:installation/installation-apache-arrow.adoc[Apache Arrow] . xref:installation/additional-config-parameters.adoc[Additional configuration options] . xref:installation/System-requirements.adoc[System Requirements] diff --git a/doc/modules/ROOT/pages/installation/installation-causal-cluster.adoc b/doc/modules/ROOT/pages/installation/installation-causal-cluster.adoc deleted file mode 100644 index c78fee8fad1..00000000000 --- a/doc/modules/ROOT/pages/installation/installation-causal-cluster.adoc +++ /dev/null @@ -1,11 +0,0 @@ -[[installation-causal-cluster]] -= Neo4j Causal Cluster - -include::partial$/common-usage/not-on-aurads-note.adoc[] - -In a Neo4j Causal Cluster, GDS should only be installed on a server that is not essential for handling transactional load. This is because the compute-intensive OLAP workloads in GDS might interfere with the smooth operation of the OLTP system that is Neo4j Causal Cluster. - -In order to install the GDS library you can follow the steps from xref:installation/neo4j-server.adoc[Neo4j Server]. -Additionally, the Neo4j Causal Cluster must be configured to use https://neo4j.com/docs/operations-manual/current/clustering/internals/#causal-clustering-routing[server-side routing]. - -For more details, see xref:production-deployment/causal-cluster.adoc[GDS with Neo4j Causal Cluster]. diff --git a/doc/modules/ROOT/pages/installation/installation-neo4j-cluster.adoc b/doc/modules/ROOT/pages/installation/installation-neo4j-cluster.adoc new file mode 100644 index 00000000000..cef70ad9eea --- /dev/null +++ b/doc/modules/ROOT/pages/installation/installation-neo4j-cluster.adoc @@ -0,0 +1,11 @@ +[[installation-neo4j-cluster]] += Neo4j cluster + +include::partial$/common-usage/not-on-aurads-note.adoc[] + +In a Neo4j cluster, GDS should only be installed on a server that is not essential for handling transactional load. This is because the compute-intensive OLAP workloads in GDS might interfere with the smooth operation of the OLTP system that is Neo4j cluster. + +In order to install the GDS library you can follow the steps from xref:installation/neo4j-server.adoc[Neo4j Server]. +Additionally, the Neo4j cluster must be configured to use https://neo4j.com/docs/operations-manual/current/clustering/setup/routing/#clustering-routing[server-side routing]. + +For more details, see xref:production-deployment/neo4j-cluster.adoc[GDS with Neo4j cluster]. diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index 0c946584f91..c9c8c9900dc 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -5,8 +5,8 @@ The GDS library is intended to be used on a standalone Neo4j server. [NOTE] ==== -Running the GDS library on a core member of a Neo4j Causal Cluster is not supported. -Read more about how to use GDS in conjunction with Neo4j Causal Cluster deployment xref:installation/installation-causal-cluster.adoc[below]. +Running the GDS library on a core member of a Neo4j cluster is not supported. +Read more about how to use GDS in conjunction with Neo4j cluster deployment xref:installation/installation-neo4j-cluster.adoc[below]. ==== On a standalone Neo4j Server, the library will need to be installed and configured manually. diff --git a/doc/modules/ROOT/pages/introduction.adoc b/doc/modules/ROOT/pages/introduction.adoc index 93a7496f956..74aedcb177b 100644 --- a/doc/modules/ROOT/pages/introduction.adoc +++ b/doc/modules/ROOT/pages/introduction.adoc @@ -97,7 +97,7 @@ The Neo4j Graph Data Science library is available in two editions. * The Neo4j Graph Data Science library Enterprise Edition: ** Can run on an unlimited amount of CPU cores. ** Supports the role-based access control system (RBAC) from Neo4j Enterprise Edition. -** Support running GDS as part of a xref::production-deployment/causal-cluster.adoc[cluster deployment]. +** Support running GDS as part of a xref::production-deployment/neo4j-cluster.adoc[Neo4j cluster deployment]. ** Includes capacity and load xref::common-usage/monitoring-system.adoc[monitoring]. ** Supports various additional graph catalog features, including: *** Graph xref::management-ops/backup-restore.adoc[backup and restore]. diff --git a/doc/modules/ROOT/pages/production-deployment/index.adoc b/doc/modules/ROOT/pages/production-deployment/index.adoc index 0b3b7911b6e..d60b2a87270 100644 --- a/doc/modules/ROOT/pages/production-deployment/index.adoc +++ b/doc/modules/ROOT/pages/production-deployment/index.adoc @@ -7,5 +7,5 @@ This chapter is divided into the following sections: * xref:production-deployment/transaction-handling.adoc[Transaction Handling] * xref:production-deployment/fabric.adoc[Using GDS and Fabric] -* xref:production-deployment/causal-cluster.adoc[GDS with Neo4j Causal Cluster] +* xref:production-deployment/neo4j-cluster.adoc[GDS with Neo4j cluster] * xref:production-deployment/feature-toggles.adoc[GDS Feature Toggles] diff --git a/doc/modules/ROOT/pages/production-deployment/causal-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc similarity index 91% rename from doc/modules/ROOT/pages/production-deployment/causal-cluster.adoc rename to doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 0e68a5870c8..2e701d1417d 100644 --- a/doc/modules/ROOT/pages/production-deployment/causal-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -1,12 +1,12 @@ [.enterprise-edition] [[cluster]] -= GDS with Neo4j Causal Cluster -:description: This section describes how the Neo4j Graph Data Science library can be used in a Neo4j Causal Cluster deployment. += GDS with Neo4j cluster +:description: This section describes how the Neo4j Graph Data Science library can be used in a Neo4j cluster deployment. include::partial$/common-usage/not-on-aurads-note.adoc[] -It is possible to run GDS as part of Neo4j Causal Cluster deployment. +It is possible to run GDS as part of Neo4j cluster deployment. Since GDS performs large computations with the full resources of the system it is not suitable to run on instances that serve the transactional workload of the cluster. @@ -15,29 +15,6 @@ Since GDS performs large computations with the full resources of the system it i [.tabbed-example, caption = ] ==== -[.include-with-neo4j-4x] -===== - -We make use of a _Read Replica_ instance to deploy the GDS library and process analytical workloads. -Calls to GDS `write` procedures are internally directed to the cluster `LEADER` instance via _server-side routing_. - -[NOTE] -====== -Please refer to the https://neo4j.com/docs/operations-manual/4.4/clustering/[official Neo4j documentation] for details on how to setup Neo4j Causal Cluster. -Note that the link points to the latest Neo4j 4.x version documentation and the configuration settings may differ from earlier versions. -====== - -* The cluster must contain at least one _Read Replica_ instance -** single _Core member_ and a _Read Replica_ is a valid scenario. -** GDS workloads are not load-balanced if there are more than one _Read Replica_ instances. -* Cluster should be configured to use https://neo4j.com/docs/operations-manual/4.4/clustering/internals/#causal-clustering-routing[server-side routing]. -* GDS plugin deployed on the _Read Replica_. -** A valid GDS Enterprise Edition license must be installed and configured on the _Read Replica_. -** The driver connection to operated GDS should be made using the `bolt://` protocol, or _server-policy routed_ to the _Read Replica_ instance. - -For more information on setting up, configuring and managing a Neo4j Causal Clustering, please refer to https://neo4j.com/docs/operations-manual/4.4/clustering/[the documentation]. -===== - [.include-with-neo4j-5x] ===== @@ -49,19 +26,19 @@ GDS should be installed on a machine that is not serving transactional load and [NOTE] ====== -Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/[official Neo4j documentation] for details on how to setup Neo4j Causal Cluster. +Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/[official Neo4j documentation] for details on how to setup Neo4j cluster. Note that the link points to the latest Neo4j version documentation and the configuration settings may differ from earlier versions. ====== * The cluster must contain at least one _Secondary_ machine ** single _Primary_ and a _Secondary_ is a valid scenario. ** GDS workloads are not load-balanced if there are more than one _Secondary_ instances. -* Cluster should be configured to use https://neo4j.com/docs/operations-manual/current/clustering/internals/#causal-clustering-routing[server-side routing]. +* Cluster should be configured to use https://neo4j.com/docs/operations-manual/current/clustering/internals/#clustering-routing[server-side routing]. * GDS plugin deployed on the _Secondary_. ** A valid GDS Enterprise Edition license must be installed and configured on the _Secondary_. ** The driver connection to operated GDS should be made using the `bolt://` protocol to the _Secondary_ instance. -For more information on setting up, configuring and managing a Neo4j Causal Clustering, please refer to https://neo4j.com/docs/operations-manual/current/clustering/[the documentation]. +For more information on setting up, configuring and managing a Neo4j clustering, please refer to https://neo4j.com/docs/operations-manual/current/clustering/[the documentation]. [NOTE] ====== @@ -74,6 +51,30 @@ However, you might not be able to reuse that same configuration file verbatim on It is of course also possible to turn strict validation off. ====== ===== + +[.include-with-neo4j-4x] +===== + +We make use of a _Read Replica_ instance to deploy the GDS library and process analytical workloads. +Calls to GDS `write` procedures are internally directed to the cluster `LEADER` instance via _server-side routing_. + +[NOTE] +====== +Please refer to the https://neo4j.com/docs/operations-manual/4.4/clustering/[official Neo4j documentation] for details on how to setup Neo4j Causal Cluster. +Note that the link points to the latest Neo4j 4.x version documentation and the configuration settings may differ from earlier versions. +====== + +* The cluster must contain at least one _Read Replica_ instance +** single _Core member_ and a _Read Replica_ is a valid scenario. +** GDS workloads are not load-balanced if there are more than one _Read Replica_ instances. +* Cluster should be configured to use https://neo4j.com/docs/operations-manual/4.4/clustering/internals/#causal-clustering-routing[server-side routing]. +* GDS plugin deployed on the _Read Replica_. +** A valid GDS Enterprise Edition license must be installed and configured on the _Read Replica_. +** The driver connection to operated GDS should be made using the `bolt://` protocol, or _server-policy routed_ to the _Read Replica_ instance. + +For more information on setting up, configuring and managing a Neo4j Causal Clustering, please refer to https://neo4j.com/docs/operations-manual/4.4/clustering/[the documentation]. +===== + ==== == GDS Configuration From 23515b33dff2370a7f39c33fe915d3bae49fd1a2 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 13 Jan 2023 10:49:06 +0000 Subject: [PATCH 003/400] Address review comments Co-authored-by: Jessica Wright --- doc/modules/ROOT/pages/installation/neo4j-server.adoc | 6 ------ .../ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index c9c8c9900dc..745577e9a80 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -3,12 +3,6 @@ The GDS library is intended to be used on a standalone Neo4j server. -[NOTE] -==== -Running the GDS library on a core member of a Neo4j cluster is not supported. -Read more about how to use GDS in conjunction with Neo4j cluster deployment xref:installation/installation-neo4j-cluster.adoc[below]. -==== - On a standalone Neo4j Server, the library will need to be installed and configured manually. 1. Download `neo4j-graph-data-science-[version].jar` from the https://neo4j.com/download-center/#algorithms[Neo4j Download Center] and copy it into the `$NEO4J_HOME/plugins` directory. diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 2e701d1417d..657541c98a3 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -38,7 +38,7 @@ Note that the link points to the latest Neo4j version documentation and the conf ** A valid GDS Enterprise Edition license must be installed and configured on the _Secondary_. ** The driver connection to operated GDS should be made using the `bolt://` protocol to the _Secondary_ instance. -For more information on setting up, configuring and managing a Neo4j clustering, please refer to https://neo4j.com/docs/operations-manual/current/clustering/[the documentation]. +For more information on setting up, configuring and managing a Neo4j cluster, please refer to https://neo4j.com/docs/operations-manual/current/clustering/[the documentation]. [NOTE] ====== From 28a0384b68ac640787062529ef015f20d441219b Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 10 Nov 2022 12:10:46 +0100 Subject: [PATCH 004/400] Replace `aurads` qualifier with a fourth version number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- gradle/version.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index b32a9106fb1..1c2eabde08d 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,3 +1,6 @@ ext { - gdsVersion = '2.3.0-alpha05' + gdsBaseVersion = '2.3.0' + gdsAuraVersion = '8' + + gdsVersion = gdsBaseVersion + (properties.getOrDefault('aurads', false) ? ".${gdsAuraVersion}" : "") } From d545f9aa9c50c5834e4a14ff60d05bb6cac66748 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 10 Nov 2022 14:34:37 +0100 Subject: [PATCH 005/400] Fix GDS version composition for AuraDS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - use a `+` instead of `.` to be semver compatible - disable doc tests for AuraDS builds - make BuildInfoPropertiesTest more liberal (accepts a suffix number) Co-authored-by: Jonatan Jäderberg --- doc-test/build.gradle | 6 +++++- gradle/version.gradle | 2 +- .../test/java/org/neo4j/gds/BuildInfoPropertiesTest.java | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/doc-test/build.gradle b/doc-test/build.gradle index 9f15dc4dbb7..20f18c81bcf 100644 --- a/doc-test/build.gradle +++ b/doc-test/build.gradle @@ -1,6 +1,6 @@ apply plugin: 'java-library' -description = 'Neo4j Graph Data Science :: Docs' +description = 'Neo4j Graph Data Science :: Documentation Testing' group = 'org.neo4j.gds' @@ -41,3 +41,7 @@ tasks.register('unpackDocs', Copy) { } processTestResources.dependsOn tasks.named('unpackDocs') + +tasks.test { + enabled = !rootProject.hasProperty('aurads') +} diff --git a/gradle/version.gradle b/gradle/version.gradle index 1c2eabde08d..37a697a5ccd 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -2,5 +2,5 @@ ext { gdsBaseVersion = '2.3.0' gdsAuraVersion = '8' - gdsVersion = gdsBaseVersion + (properties.getOrDefault('aurads', false) ? ".${gdsAuraVersion}" : "") + gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java index 93521804ba4..5776b18d176 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/BuildInfoPropertiesTest.java @@ -50,7 +50,7 @@ void shouldReturnGradleVersion() throws IOException { fail("Could not find version in file: " + file.toAbsolutePath())); var buildInfo = BuildInfoProperties.get(); - assertEquals(version, buildInfo.gdsVersion()); + assertThat(buildInfo.gdsVersion()).startsWith(version); } @Test @@ -106,7 +106,7 @@ void loadFromPropertiesRequiresVersion() { } private Optional findVersion(Path file) throws IOException { - Pattern pattern = Pattern.compile(".*gdsVersion = '(\\d\\.\\d\\.\\d+(-alpha\\d+|-beta\\d+)?)'.*"); + Pattern pattern = Pattern.compile(".*gdsBaseVersion = '(\\d\\.\\d\\.\\d+(-alpha\\d+|-beta\\d+)?)'.*"); return Files.lines(file, StandardCharsets.UTF_8) .flatMap(line -> { var matcher = pattern.matcher(line); From f1a33f94066847bc21631d030c9fd0f133a20496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 13 Jan 2023 13:47:38 +0100 Subject: [PATCH 006/400] Update links to pregel source code --- doc/modules/ROOT/pages/algorithms/pregel-api.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc index 66451f79d3b..be357ed0537 100644 --- a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc +++ b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc @@ -29,7 +29,7 @@ The introduction of a new Pregel algorithm can be separated in two main steps. First, we need to implement the algorithm using the Pregel Java API. Second, we need to expose the algorithm via a Cypher procedure to make use of it. -For an example on how to expose a custom Pregel computation via a Neo4j procedure, have a look at the https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-example/src/main/java/org/neo4j/gds/beta/pregel[Pregel examples]. +For an example on how to expose a custom Pregel computation via a Neo4j procedure, have a look at the https://github.com/neo4j/graph-data-science/tree/2.3/examples/pregel-example/src/main/java/org/neo4j/gds/beta/pregel[Pregel examples]. [[algorithms-pregel-api-java]] == Pregel Java API @@ -510,7 +510,7 @@ For more details, please refer to the xref:algorithms/pregel-api.adoc#algorithms === Building and installing a Neo4j plugin In order to use a Pregel algorithm in Neo4j via a procedure, we need to package it as Neo4j plugin. -The https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-bootstrap[pregel-bootstrap] project is a good starting point. +The https://github.com/neo4j/graph-data-science/tree/2.3/examples/pregel-bootstrap[pregel-bootstrap] project is a good starting point. The `build.gradle` file within the project contains all the dependencies necessary to implement a Pregel algorithm and to generate corresponding procedures. Make sure to change the `gdsVersion` and `neo4jVersion` according to your setup. @@ -541,7 +541,7 @@ NOTE: Before `Neo4j 4.2`, the configuration setting is called `dbms.security.pro [[algorithms-pregel-api-example]] == Examples -The https://github.com/neo4j/graph-data-science/tree/master/examples/pregel-example[pregel-examples] module contains a set of examples for Pregel algorithms. +The https://github.com/neo4j/graph-data-science/tree/2.3/examples/pregel-example[pregel-examples] module contains a set of examples for Pregel algorithms. The algorithm implementations demonstrate the usage of the Pregel API. Along with each example, we provide test classes that can be used as a guideline on how to write tests for custom algorithms. To play around, we recommend copying one of the algorithms into the `pregel-bootstrap` project, build it and setup the plugin in Neo4j. From caaada5b2625baf12a7f4a9d2b14d98cb8c6c37b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 12 Jan 2023 17:04:11 +0100 Subject: [PATCH 007/400] Validate concurrency in GraphStreamRelationshipsConfig --- .../GraphStreamRelationshipsConfig.java | 9 +--- .../GraphStreamRelationshipsConfigTest.java | 41 +++++++++++++++++++ .../GraphStreamRelationshipsProcTest.java | 2 +- 3 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 core/src/test/java/org/neo4j/gds/config/GraphStreamRelationshipsConfigTest.java diff --git a/core/src/main/java/org/neo4j/gds/config/GraphStreamRelationshipsConfig.java b/core/src/main/java/org/neo4j/gds/config/GraphStreamRelationshipsConfig.java index 720abc3cf6a..584ffff1cd9 100644 --- a/core/src/main/java/org/neo4j/gds/config/GraphStreamRelationshipsConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/GraphStreamRelationshipsConfig.java @@ -23,7 +23,6 @@ import org.neo4j.gds.ElementProjection; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.Configuration; -import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.core.CypherMapWrapper; @@ -36,9 +35,8 @@ import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -@ValueClass @Configuration -public interface GraphStreamRelationshipsConfig extends BaseConfig { +public interface GraphStreamRelationshipsConfig extends BaseConfig, ConcurrencyConfig { @Configuration.Parameter Optional graphName(); @@ -49,11 +47,6 @@ default List relationshipTypes() { return Collections.singletonList(ElementProjection.PROJECT_ALL); } - @Value.Default - default int concurrency() { - return ConcurrencyConfig.DEFAULT_CONCURRENCY; - } - @Configuration.Ignore default Collection relationshipTypeIdentifiers(GraphStore graphStore) { return relationshipTypes().contains(ElementProjection.PROJECT_ALL) diff --git a/core/src/test/java/org/neo4j/gds/config/GraphStreamRelationshipsConfigTest.java b/core/src/test/java/org/neo4j/gds/config/GraphStreamRelationshipsConfigTest.java new file mode 100644 index 00000000000..25dab8ff9a8 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/config/GraphStreamRelationshipsConfigTest.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.config; + +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class GraphStreamRelationshipsConfigTest { + + @Test + void validateConcurrency() { + var builder = GraphStreamRelationshipsConfigImpl.builder() + .graphName(Optional.of("g")) + .relationshipTypes(List.of("*")) + .concurrency(42); + + assertThatThrownBy(builder::build).hasMessageContaining("Community users cannot exceed concurrency"); + } + +} diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java index ac51b00339b..82e28ff9cdd 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java @@ -133,7 +133,7 @@ void shouldStreamInParallel() { runQuery("CALL gds.beta.graph.generate('generatedGraph', 10000, 5)"); var actualRelationships = new ArrayList(); - runQueryWithRowConsumer("CALL gds.beta.graph.relationships.stream('generatedGraph', ['*'], { concurrency: 4 })", row -> actualRelationships.add( + runQueryWithRowConsumer("CALL gds.beta.graph.relationships.stream('generatedGraph', ['*'], { concurrency: 10 })", row -> actualRelationships.add( relationship( row.getNumber("sourceNodeId").longValue(), row.getNumber("targetNodeId").longValue(), From 032d42b253d41ec9158b26ffacbce2a1d00e313c Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 13 Jan 2023 14:47:00 +0100 Subject: [PATCH 008/400] Use a valid concurrency for a public test --- .../org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java index 82e28ff9cdd..ac51b00339b 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipsProcTest.java @@ -133,7 +133,7 @@ void shouldStreamInParallel() { runQuery("CALL gds.beta.graph.generate('generatedGraph', 10000, 5)"); var actualRelationships = new ArrayList(); - runQueryWithRowConsumer("CALL gds.beta.graph.relationships.stream('generatedGraph', ['*'], { concurrency: 10 })", row -> actualRelationships.add( + runQueryWithRowConsumer("CALL gds.beta.graph.relationships.stream('generatedGraph', ['*'], { concurrency: 4 })", row -> actualRelationships.add( relationship( row.getNumber("sourceNodeId").longValue(), row.getNumber("targetNodeId").longValue(), From f9311256f03ef2f233d960e06bd533ec3f29d84d Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Mon, 16 Jan 2023 07:04:05 +0000 Subject: [PATCH 009/400] Update System Requirements section in Installation --- .../installation/System-requirements.adoc | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index fa727199f02..43e86731392 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -15,9 +15,24 @@ image::memory-usage.png[width=600] === Heap size The heap space is used for storing graph projections in the graph catalog and algorithm state. + +[.tabbed-example, caption = ] +==== + +[.include-with-neo4j-5x] +===== +When writing algorithm results back to Neo4j, heap space is also used for handling transaction state (see https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#config_db.tx_state.memory_allocation[dbms.tx_state.memory_allocation]). +For purely analytical workloads, a general recommendation is to set the heap space to about 90% of the available main memory. +This can be done via https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#config_server.memory.heap.initial_size[server.memory.heap.initial_size] and https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#config_server.memory.heap.max_size[server.memory.heap.max_size]. +===== + +[.include-with-neo4j-4x] +===== When writing algorithm results back to Neo4j, heap space is also used for handling transaction state (see https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.tx_state.memory_allocation[dbms.tx_state.memory_allocation]). For purely analytical workloads, a general recommendation is to set the heap space to about 90% of the available main memory. This can be done via https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.heap.initial_size[dbms.memory.heap.initial_size] and https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.heap.max_size[dbms.memory.heap.max_size]. +===== +==== To better estimate the heap space required to project graphs and run algorithms, consider the xref:common-usage/memory-estimation.adoc[Memory Estimation] feature. The feature estimates the memory consumption of all involved data structures using information about number of nodes and relationships from the Neo4j count store. @@ -26,7 +41,24 @@ The feature estimates the memory consumption of all involved data structures usi The page cache is used to cache the Neo4j data and will help to avoid costly disk access. -For purely analytical workloads including xref:management-ops/projections/graph-project.adoc[native projections], it is recommended to decrease https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.pagecache.size[dbms.memory.pagecache.size] in favor of an increased heap size. +For purely analytical workloads including xref:management-ops/projections/graph-project.adoc[native projections], it is recommended to decrease the configured PageCache in favor of an increased heap size. + +[.tabbed-example, caption = ] +==== + +[.include-with-neo4j-5x] +===== +To configure the PageCache size you can use the following Neo4j configuration property +https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#config_server.memory.pagecache.size[server.memory.pagecache.size] +===== + +[.include-with-neo4j-4x] +===== +To configure the PageCache size you can use the following Neo4j configuration property +https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.pagecache.size[dbms.memory.pagecache.size] +===== +==== + However, setting a minimum page cache size is still important when projecting graphs: * For xref:management-ops/projections/graph-project.adoc[native projections], the minimum page cache size for projecting a graph can be roughly estimated by `8KB * 100 * readConcurrency`. From ee0a2a6cafc32c561977d68deb4dc8742939df41 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Mon, 16 Jan 2023 09:45:25 +0000 Subject: [PATCH 010/400] Address review comments Co-authored-by: Mats Rydberg --- .../pages/installation/System-requirements.adoc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index 43e86731392..5b1db524424 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -1,11 +1,9 @@ [[System-requirements]] = System Requirements -:neo4j-docs-link-version: 4.4 - == Main Memory -The GDS library runs within a Neo4j instance and is therefore subject to the general https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/performance/memory-configuration/[Neo4j memory configuration]. +The GDS library runs within a Neo4j instance and is therefore subject to the general https://neo4j.com/docs/operations-manual/4.4/performance/memory-configuration/[Neo4j memory configuration]. .GDS heap memory usage image::memory-usage.png[width=600] @@ -14,7 +12,7 @@ image::memory-usage.png[width=600] [[heap-size]] === Heap size -The heap space is used for storing graph projections in the graph catalog and algorithm state. +The heap space is used for storing graph projections in the graph catalog, and algorithm state. [.tabbed-example, caption = ] ==== @@ -28,9 +26,9 @@ This can be done via https://neo4j.com/docs/operations-manual/5/reference/config [.include-with-neo4j-4x] ===== -When writing algorithm results back to Neo4j, heap space is also used for handling transaction state (see https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.tx_state.memory_allocation[dbms.tx_state.memory_allocation]). +When writing algorithm results back to Neo4j, heap space is also used for handling transaction state (see https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#config_dbms.tx_state.memory_allocation[dbms.tx_state.memory_allocation]). For purely analytical workloads, a general recommendation is to set the heap space to about 90% of the available main memory. -This can be done via https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.heap.initial_size[dbms.memory.heap.initial_size] and https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.heap.max_size[dbms.memory.heap.max_size]. +This can be done via https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#config_dbms.memory.heap.initial_size[dbms.memory.heap.initial_size] and https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#config_dbms.memory.heap.max_size[dbms.memory.heap.max_size]. ===== ==== @@ -55,7 +53,7 @@ https://neo4j.com/docs/operations-manual/5/reference/configuration-settings/#con [.include-with-neo4j-4x] ===== To configure the PageCache size you can use the following Neo4j configuration property -https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/reference/configuration-settings/#config_dbms.memory.pagecache.size[dbms.memory.pagecache.size] +https://neo4j.com/docs/operations-manual/4.4/reference/configuration-settings/#config_dbms.memory.pagecache.size[dbms.memory.pagecache.size] ===== ==== @@ -71,7 +69,7 @@ Ideally, if the xref:common-usage/memory-estimation.adoc[memory estimation] feat [NOTE] ==== Decreasing the page cache size in favor of heap size is *not* recommended if the Neo4j instance runs both, operational and analytical workloads at the same time. -See https://neo4j.com/docs/operations-manual/{neo4j-docs-link-version}/performance/memory-configuration/[Neo4j memory configuration] for general information about page cache sizing. +See https://neo4j.com/docs/operations-manual/4.4/performance/memory-configuration/[Neo4j memory configuration] for general information about page cache sizing. ==== [[system-requirements-cpu]] From 5f2cd00ecc7afb6357e016c4f02f2ebdfe1becb1 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 16 Jan 2023 13:55:35 +0100 Subject: [PATCH 011/400] Fix test assertion following version bump Co-authored-by: Brian Shi --- doc/modules/ROOT/pages/management-ops/utility-functions.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 2dde5be0fc4..a4f84b26c78 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.0-alpha05" +| "2.3.0" |=== -- From 51adeb926506d75cbcd636916bca047f713c0cc6 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 16 Jan 2023 17:36:03 +0100 Subject: [PATCH 012/400] Remove docs publishing from maintenance branch --- doc/package.json | 3 --- doc/publish.yml | 60 ------------------------------------------------ 2 files changed, 63 deletions(-) delete mode 100644 doc/publish.yml diff --git a/doc/package.json b/doc/package.json index 672e5ff5c5d..7e279854a06 100644 --- a/doc/package.json +++ b/doc/package.json @@ -6,12 +6,9 @@ "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "start": "nodemon -e adoc --exec \"npm run build && npm run serve\"", - "start:publish": "nodemon -e adoc --exec \"npm run build:publish && npm run serve\"", "serve": "node server.js", "build": "antora preview.yml --stacktrace --log-format=pretty", - "build:publish": "antora publish.yml --stacktrace --log-format=pretty", "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json", - "publish-verify": "antora --stacktrace --fetch publish.yml --log-format=json --log-file ./build/log/log.json" }, "keywords": [ "antora", diff --git a/doc/publish.yml b/doc/publish.yml deleted file mode 100644 index 09341aeb3b8..00000000000 --- a/doc/publish.yml +++ /dev/null @@ -1,60 +0,0 @@ -site: - title: Neo4j Graph Data Science - url: https://neo4j.com/docs - start_page: graph-data-science:ROOT:index.adoc - -content: - sources: - - url: https://github.com/neo-technology/graph-analytics.git - branches: ['2.2', 'master'] - start_path: public/doc - include: public/doc/ - exclude: - - '!**/_includes/*' - - '!**/readme.adoc' - - '!**/README.adoc' -ui: - bundle: - url: https://s3-eu-west-1.amazonaws.com/static-content.neo4j.com/build/ui-bundle-latest.zip - snapshot: true - output_dir: /assets - -urls: - html_extension_style: indexify - -antora: - extensions: - - require: "@neo4j-antora/antora-modify-sitemaps" - sitemap_version: '2.2' - sitemap_loc_version: current - move_sitemaps_to_components: 'true' - -asciidoc: - extensions: - - "@neo4j-documentation/remote-include" - - "@neo4j-documentation/macros" - - "@neo4j-antora/antora-page-roles" - - "@neo4j-antora/antora-table-footnotes" - attributes: - page-theme: docs - page-type: Docs - page-search-type: Docs - page-search-site: Reference Docs - page-canonical-root: /docs - page-pagination: true - page-no-canonical: true - page-origin-private: true - page-hide-toc: false - page-mixpanel: 4bfb2414ab973c741b6f067bf06d5575 - # page-cdn: /static/assets - includePDF: false - sectnums: true - sectnumlevel: 3 - doctype: book - nonhtmloutput: "" - # sectnums: true, removed so they are off by default - # sectnumlevel: 3, - experimental: '' - copyright: 2022 - common-license-page-uri: https://neo4j.com/docs/license/ - operations-manual-base-uri: https://neo4j.com/docs/operations-manual/ From 1cf0ebb602f058b14d59c244178575e1d552ae20 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 16 Jan 2023 17:42:39 +0100 Subject: [PATCH 013/400] Avoid calling it Library --- doc/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/package.json b/doc/package.json index 7e279854a06..fb566faf308 100644 --- a/doc/package.json +++ b/doc/package.json @@ -1,7 +1,7 @@ { "name": "graph-data-science", "version": "2.3.0-preview", - "description": "Neo4j Graph Data Science Library", + "description": "Neo4j Graph Data Science", "main": "server.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", From 7514d1072483369604914061dff79b46e37e24f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Fri, 13 Jan 2023 10:18:17 +0100 Subject: [PATCH 014/400] Add validation methods for inverse indexes to PregelBaseProc --- proc/pregel/build.gradle | 2 +- .../neo4j/gds/pregel/proc/PregelBaseProc.java | 103 +++++++++++++++--- .../gds/pregel/proc/PregelBaseProcTest.java | 86 +++++++++++++++ 3 files changed, 175 insertions(+), 16 deletions(-) create mode 100644 proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java diff --git a/proc/pregel/build.gradle b/proc/pregel/build.gradle index a3b821a38d6..babfad1e869 100644 --- a/proc/pregel/build.gradle +++ b/proc/pregel/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation project(':executor') implementation project(':proc-common') implementation project(':string-formatting') - + implementation project(':graph-schema-api') api project(':pregel') testAnnotationProcessor project(':annotations') diff --git a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java index fdf2582a45e..dd4dea0fa47 100644 --- a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java +++ b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java @@ -20,34 +20,109 @@ package org.neo4j.gds.pregel.proc; import org.neo4j.gds.Algorithm; +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.nodes.DoubleArrayNodePropertyValues; import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.beta.indexInverse.InverseRelationshipsAlgorithmFactory; +import org.neo4j.gds.beta.indexInverse.InverseRelationshipsConfigImpl; import org.neo4j.gds.beta.pregel.PregelConfig; import org.neo4j.gds.beta.pregel.PregelResult; import org.neo4j.gds.beta.pregel.PregelSchema; import org.neo4j.gds.core.utils.paged.HugeObjectArray; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; import org.neo4j.gds.core.write.ImmutableNodeProperty; import org.neo4j.gds.core.write.NodeProperty; import org.neo4j.gds.executor.ComputationResult; +import org.neo4j.gds.executor.validation.AfterLoadValidation; +import org.neo4j.gds.executor.validation.ValidationConfiguration; +import org.neo4j.gds.utils.StringJoining; +import org.neo4j.logging.Log; +import java.util.Collection; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -final class PregelBaseProc { +public final class PregelBaseProc { - static , CONFIG extends PregelConfig> - List nodeProperties( - ComputationResult computationResult, - String propertyPrefix + public static ValidationConfiguration ensureIndexValidation( + Log log, TaskRegistryFactory taskRegistryFactory + ) { + return new ValidationConfiguration<>() { + @Override + public List> afterLoadValidations() { + return List.of( + (graphStore, graphProjectConfig, config) -> ensureDirectedRelationships( + graphStore, config.internalRelationshipTypes(graphStore) + ), + (graphStore, graphProjectConfig, config) -> ensureInverseIndexesExist(graphStore, + config.internalRelationshipTypes(graphStore), + config.concurrency(), + log, + taskRegistryFactory + ) + ); + } + }; + } + + static void ensureInverseIndexesExist( + GraphStore graphStore, + Collection relationshipTypes, + int concurrency, + Log log, + TaskRegistryFactory taskRegistryFactory + ) { + relationshipTypes + .stream() + .filter(relType -> !graphStore.inverseIndexedRelationshipTypes().contains(relType)) + .forEach(relType -> { + var inverseConfig = InverseRelationshipsConfigImpl + .builder() + .concurrency(concurrency) + .relationshipType(relType.name) + .build(); + + var inverseIndex = new InverseRelationshipsAlgorithmFactory() + .build(graphStore, inverseConfig, log, taskRegistryFactory) + .compute(); + + graphStore.addInverseIndex(relType, inverseIndex); + }); + } + + static void ensureDirectedRelationships( + GraphStore graphStore, Collection relationshipTypes + ) { + var relationshipSchema = graphStore.schema().relationshipSchema(); + var undirectedTypes = relationshipTypes + .stream() + .filter(relationshipSchema::isUndirected) + .map(RelationshipType::name) + .collect(Collectors.toList()); + + if (!undirectedTypes.isEmpty()) { + throw new IllegalArgumentException(String.format( + Locale.US, + "This algorithm requires a directed graph, but the following configured relationship types are undirected: %s.", + StringJoining.join(undirectedTypes) + )); + } + } + + static , CONFIG extends PregelConfig> List nodeProperties( + ComputationResult computationResult, String propertyPrefix ) { var compositeNodeValue = computationResult.result().nodeValues(); var schema = compositeNodeValue.schema(); // TODO change this to generic prefix setting - return schema.elements() + return schema + .elements() .stream() .filter(element -> element.visibility() == PregelSchema.Visibility.PUBLIC) .map(element -> { @@ -62,24 +137,22 @@ List nodeProperties( nodePropertyValues = compositeNodeValue.doubleProperties(propertyKey).asNodeProperties(); break; case LONG_ARRAY: - nodePropertyValues = new HugeObjectArrayLongArrayPropertyValues( - compositeNodeValue.longArrayProperties(propertyKey) - ); + nodePropertyValues = new HugeObjectArrayLongArrayPropertyValues(compositeNodeValue.longArrayProperties( + propertyKey)); break; case DOUBLE_ARRAY: - nodePropertyValues = new HugeObjectArrayDoubleArrayPropertyValues( - compositeNodeValue.doubleArrayProperties(propertyKey) - ); + nodePropertyValues = new HugeObjectArrayDoubleArrayPropertyValues(compositeNodeValue.doubleArrayProperties( + propertyKey)); break; default: throw new IllegalArgumentException("Unsupported property type: " + element.propertyType()); } - return ImmutableNodeProperty.of( - formatWithLocale("%s%s", propertyPrefix, propertyKey), + return ImmutableNodeProperty.of(formatWithLocale("%s%s", propertyPrefix, propertyKey), nodePropertyValues ); - }).collect(Collectors.toList()); + }) + .collect(Collectors.toList()); } private PregelBaseProc() {} diff --git a/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java b/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java new file mode 100644 index 00000000000..7de425bfd34 --- /dev/null +++ b/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.pregel.proc; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; +import org.neo4j.logging.NullLog; + +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@GdlExtension +class PregelBaseProcTest { + + + @GdlGraph + @GdlGraph(graphNamePrefix = "undirected", orientation = Orientation.UNDIRECTED) + public static String GDL = "CREATE " + " ()-[:REL]->()," + " ()-[:REL2]->(),"; + + + @Inject + GraphStore graphStore; + + @Inject + GraphStore undirectedGraphStore; + + static Stream relTypeCombinations() { + var rel = RelationshipType.of("REL"); + var rel2 = RelationshipType.of("REL2"); + return Stream.of(Arguments.arguments(List.of(rel)), + Arguments.arguments(List.of(rel2)), + Arguments.arguments(List.of(rel, rel2)) + ); + } + + @ParameterizedTest + @MethodSource("relTypeCombinations") + void shouldGenerateInverseIndexes(List relTypes) { + PregelBaseProc.ensureInverseIndexesExist(graphStore, + relTypes, + 4, + NullLog.getInstance(), + TaskRegistryFactory.empty() + ); + assertThat(graphStore.inverseIndexedRelationshipTypes()).containsExactlyElementsOf(relTypes); + } + + @Test + void shouldThrowOnUndirectedRelTypes() { + assertThatThrownBy(() -> PregelBaseProc.ensureDirectedRelationships( + undirectedGraphStore, + RelationshipType.listOf("REL", "REL2") + )).hasMessage( + "This algorithm requires a directed graph, but the following configured relationship types are undirected: ['REL', 'REL2']." + ); + } +} From 45025575d7a7ed34fe7aa20c021c303fef2b6e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Fri, 13 Jan 2023 10:18:50 +0100 Subject: [PATCH 015/400] Add inverse index validations to generated Pregel procs --- .../gds/beta/pregel/PregelValidation.java | 28 ++++-- .../gds/beta/pregel/ProcedureGenerator.java | 19 ++++ .../gds/beta/pregel/PregelProcessorTest.java | 16 +++ .../BidirectionalComputationAlgorithm.java | 38 +++++++ .../BidirectionalComputationStreamProc.java | 99 +++++++++++++++++++ .../positive/BidirectionalComputation.java | 48 +++++++++ 6 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java create mode 100644 pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java create mode 100644 pregel-proc-generator/src/test/resources/positive/BidirectionalComputation.java diff --git a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java index d129113bc7b..5d218909d6e 100644 --- a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java +++ b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java @@ -50,7 +50,10 @@ final class PregelValidation { private final Elements elementUtils; // Represents the PregelComputation interface - private final TypeMirror pregelComputation; + private final TypeMirror basePregelComputation; + + private final TypeMirror bidirectionalPregelComputation; + // Represents the PregelProcedureConfig interface private final TypeMirror pregelProcedureConfig; @@ -58,9 +61,12 @@ final class PregelValidation { this.messager = messager; this.typeUtils = typeUtils; this.elementUtils = elementUtils; - this.pregelComputation = MoreTypes.asDeclared( + this.basePregelComputation = MoreTypes.asDeclared( typeUtils.erasure(elementUtils.getTypeElement(BasePregelComputation.class.getName()).asType()) ); + this.bidirectionalPregelComputation = MoreTypes.asDeclared( + typeUtils.erasure(elementUtils.getTypeElement(BidirectionalPregelComputation.class.getName()).asType()) + ); this.pregelProcedureConfig = MoreTypes.asDeclared(elementUtils .getTypeElement(PregelProcedureConfig.class.getName()) .asType()); @@ -92,7 +98,8 @@ Optional validate(Element pregelElement) { configTypeName, procedure.name(), procedure.modes(), - maybeDescription + maybeDescription, + requiresInverseIndex(pregelElement) )); } @@ -108,17 +115,17 @@ private boolean isClass(Element pregelElement) { return isClass; } - private Optional pregelComputation(Element pregelElement) { + private Optional pregelComputation(Element pregelElement, TypeMirror computationInterface) { // TODO: this check needs to bubble up the inheritance tree return MoreElements.asType(pregelElement).getInterfaces().stream() .map(MoreTypes::asDeclared) - .filter(declaredType -> typeUtils.isSubtype(declaredType, pregelComputation)) + .filter(declaredType -> typeUtils.isSubtype(declaredType, computationInterface)) .findFirst(); } private boolean isPregelComputation(Element pregelElement) { var pregelTypeElement = MoreElements.asType(pregelElement); - var maybeInterface = pregelComputation(pregelElement); + var maybeInterface = pregelComputation(pregelElement, basePregelComputation); boolean isPregelComputation = maybeInterface.isPresent(); if (!isPregelComputation) { @@ -131,6 +138,11 @@ private boolean isPregelComputation(Element pregelElement) { return isPregelComputation; } + private boolean requiresInverseIndex(Element pregelElement) { + var maybeInterface = pregelComputation(pregelElement, bidirectionalPregelComputation); + return maybeInterface.isPresent(); + } + private boolean isPregelProcedureConfig(Element pregelElement) { var config = config(pregelElement); @@ -198,7 +210,7 @@ private boolean configHasFactoryMethod(Element pregelElement) { } private TypeMirror config(Element pregelElement) { - return pregelComputation(pregelElement) + return pregelComputation(pregelElement, basePregelComputation) .map(declaredType -> declaredType.getTypeArguments().get(0)) .orElseThrow(() -> new IllegalStateException("Could not find a pregel computation")); } @@ -218,6 +230,8 @@ interface Spec { GDSMode[] procedureModes(); Optional description(); + + boolean requiresInverseIndex(); } } diff --git a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/ProcedureGenerator.java b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/ProcedureGenerator.java index e0dad2c1e34..537c0b52fd9 100644 --- a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/ProcedureGenerator.java +++ b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/ProcedureGenerator.java @@ -36,6 +36,8 @@ import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.executor.ExecutionMode; import org.neo4j.gds.executor.GdsCallable; +import org.neo4j.gds.executor.validation.ValidationConfiguration; +import org.neo4j.gds.pregel.proc.PregelBaseProc; import org.neo4j.gds.results.MemoryEstimateResult; import org.neo4j.procedure.Description; import org.neo4j.procedure.Mode; @@ -98,6 +100,11 @@ TypeSpec typeSpec() { typeSpecBuilder.addMethod(procResultMethod()); typeSpecBuilder.addMethod(newConfigMethod()); typeSpecBuilder.addMethod(algorithmFactoryMethod(algorithmClassName)); + + if (pregelSpec.requiresInverseIndex()) { + typeSpecBuilder.addMethod(validationConfigMethod()); + } + return typeSpecBuilder.build(); } @@ -281,4 +288,16 @@ private MethodSpec algorithmFactoryMethod(ClassName algorithmClassName) { .addStatement("return $L", anonymousFactoryType) .build(); } + + private MethodSpec validationConfigMethod() { + return MethodSpec.methodBuilder("validationConfig") + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .returns(ParameterizedTypeName.get( + ClassName.get(ValidationConfiguration.class), + pregelSpec.configTypeName() + )) + .addStatement("return $T.ensureIndexValidation(log, taskRegistryFactory)", PregelBaseProc.class) + .build(); + } } diff --git a/pregel-proc-generator/src/test/java/org/neo4j/gds/beta/pregel/PregelProcessorTest.java b/pregel-proc-generator/src/test/java/org/neo4j/gds/beta/pregel/PregelProcessorTest.java index 532c29fefe9..51b87795800 100644 --- a/pregel-proc-generator/src/test/java/org/neo4j/gds/beta/pregel/PregelProcessorTest.java +++ b/pregel-proc-generator/src/test/java/org/neo4j/gds/beta/pregel/PregelProcessorTest.java @@ -62,6 +62,22 @@ void positiveTest(String className) { ); } + @ParameterizedTest + @ValueSource(strings = { + "BidirectionalComputation" + }) + void positiveBiTest(String className) { + assertAbout(javaSource()) + .that(forResource(String.format("positive/%s.java", className))) + .processedWith(new PregelProcessor()) + .compilesWithoutError() + .and() + .generatesSources( + loadExpectedFile(formatWithLocale("expected/%sStreamProc.java", className)), + loadExpectedFile(formatWithLocale("expected/%sAlgorithm.java", className)) + ); + } + @Test void baseClassMustBeAClass() { runNegativeTest( diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java new file mode 100644 index 00000000000..742dc2e3132 --- /dev/null +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java @@ -0,0 +1,38 @@ +package org.neo4j.gds.beta.pregel.cc; + +import javax.annotation.processing.Generated; +import org.neo4j.gds.Algorithm; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.beta.pregel.Pregel; +import org.neo4j.gds.beta.pregel.PregelProcedureConfig; +import org.neo4j.gds.beta.pregel.PregelResult; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.utils.TerminationFlag; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; + +@Generated("org.neo4j.gds.beta.pregel.PregelProcessor") +public final class BidirectionalComputationAlgorithm extends Algorithm { + private final Pregel pregelJob; + + BidirectionalComputationAlgorithm(Graph graph, PregelProcedureConfig configuration, + ProgressTracker progressTracker) { + super(progressTracker); + this.pregelJob = Pregel.create(graph, configuration, new BidirectionalComputation(), Pools.DEFAULT, progressTracker); + } + + @Override + public void setTerminationFlag(TerminationFlag terminationFlag) { + super.setTerminationFlag(terminationFlag); + pregelJob.setTerminationFlag(terminationFlag); + } + + @Override + public PregelResult compute() { + return pregelJob.run(); + } + + @Override + public void release() { + pregelJob.release(); + } +} diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java new file mode 100644 index 00000000000..ab0866ea5e8 --- /dev/null +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java @@ -0,0 +1,99 @@ +package org.neo4j.gds.beta.pregel.cc; + +import java.util.Map; +import java.util.stream.Stream; +import javax.annotation.processing.Generated; +import org.neo4j.gds.BaseProc; +import org.neo4j.gds.GraphAlgorithmFactory; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.beta.pregel.Pregel; +import org.neo4j.gds.beta.pregel.PregelProcedureConfig; +import org.neo4j.gds.core.CypherMapWrapper; +import org.neo4j.gds.core.utils.mem.MemoryEstimation; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.Task; +import org.neo4j.gds.executor.ExecutionMode; +import org.neo4j.gds.executor.GdsCallable; +import org.neo4j.gds.executor.validation.ValidationConfiguration; +import org.neo4j.gds.pregel.proc.PregelBaseProc; +import org.neo4j.gds.pregel.proc.PregelStreamProc; +import org.neo4j.gds.pregel.proc.PregelStreamResult; +import org.neo4j.gds.results.MemoryEstimateResult; +import org.neo4j.procedure.Description; +import org.neo4j.procedure.Mode; +import org.neo4j.procedure.Name; +import org.neo4j.procedure.Procedure; + +@GdsCallable( + name = "gds.pregel.bidirectionalTest.stream", + executionMode = ExecutionMode.STREAM, + description = "Bidirectional Test computation description" +) +@Generated("org.neo4j.gds.beta.pregel.PregelProcessor") +public final class BidirectionalComputationStreamProc extends PregelStreamProc { + @Procedure( + name = "gds.pregel.bidirectionalTest.stream", + mode = Mode.READ + ) + @Description("Bidirectional Test computation description") + public Stream stream(@Name("graphName") String graphName, + @Name(value = "configuration", defaultValue = "{}") Map configuration) { + return stream(compute(graphName, configuration)); + } + + @Procedure( + name = "gds.pregel.bidirectionalTest.stream.estimate", + mode = Mode.READ + ) + @Description(BaseProc.ESTIMATE_DESCRIPTION) + public Stream estimate( + @Name("graphNameOrConfiguration") Object graphNameOrConfiguration, + @Name("algoConfiguration") Map algoConfiguration) { + return computeEstimate(graphNameOrConfiguration, algoConfiguration); + } + + @Override + protected PregelStreamResult streamResult(long originalNodeId, long internalNodeId, + NodePropertyValues nodePropertyValues) { + throw new UnsupportedOperationException(); + } + + @Override + protected PregelProcedureConfig newConfig(String username, CypherMapWrapper config) { + return PregelProcedureConfig.of(config); + } + + @Override + public GraphAlgorithmFactory algorithmFactory( + ) { + return new GraphAlgorithmFactory() { + @Override + public BidirectionalComputationAlgorithm build(Graph graph, + PregelProcedureConfig configuration, ProgressTracker progressTracker) { + return new BidirectionalComputationAlgorithm(graph, configuration, progressTracker); + } + + @Override + public String taskName() { + return BidirectionalComputationAlgorithm.class.getSimpleName(); + } + + @Override + public Task progressTask(Graph graph, PregelProcedureConfig configuration) { + return Pregel.progressTask(graph, configuration); + } + + @Override + public MemoryEstimation memoryEstimation(PregelProcedureConfig configuration) { + var computation = new BidirectionalComputation(); + return Pregel.memoryEstimation(computation.schema(configuration), computation.reducer().isEmpty(), configuration.isAsynchronous()); + } + }; + } + + @Override + public ValidationConfiguration validationConfig() { + return PregelBaseProc.ensureIndexValidation(log, taskRegistryFactory); + } +} diff --git a/pregel-proc-generator/src/test/resources/positive/BidirectionalComputation.java b/pregel-proc-generator/src/test/resources/positive/BidirectionalComputation.java new file mode 100644 index 00000000000..1d84ceefe2c --- /dev/null +++ b/pregel-proc-generator/src/test/resources/positive/BidirectionalComputation.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.beta.pregel.cc; + +import org.neo4j.gds.beta.pregel.BidirectionalPregelComputation; +import org.neo4j.gds.beta.pregel.Messages; +import org.neo4j.gds.beta.pregel.PregelComputation; +import org.neo4j.gds.beta.pregel.PregelProcedureConfig; +import org.neo4j.gds.beta.pregel.context.ComputeContext; +import org.neo4j.gds.beta.pregel.PregelSchema; +import org.neo4j.gds.beta.pregel.annotation.GDSMode; +import org.neo4j.gds.beta.pregel.annotation.PregelProcedure; +import org.neo4j.gds.beta.pregel.context.ComputeContext.BidirectionalComputeContext; + +@PregelProcedure( + name = "gds.pregel.bidirectionalTest", + description = "Bidirectional Test computation description", + modes = {GDSMode.STREAM} +) +public class BidirectionalComputation implements BidirectionalPregelComputation { + + @Override + public PregelSchema schema(PregelProcedureConfig config) { + return null; + } + + @Override + public void compute(BidirectionalComputeContext context, Messages messages) { + + } +} From c7900bc808d32ee957858561dde309cacb821aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Fri, 13 Jan 2023 10:56:23 +0100 Subject: [PATCH 016/400] Add licenses to test expectations --- .../BidirectionalComputationAlgorithm.java | 19 +++++++++++++++++++ .../BidirectionalComputationStreamProc.java | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java index 742dc2e3132..87f984229c4 100644 --- a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationAlgorithm.java @@ -1,3 +1,22 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package org.neo4j.gds.beta.pregel.cc; import javax.annotation.processing.Generated; diff --git a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java index ab0866ea5e8..f1d3d2eea12 100644 --- a/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java +++ b/pregel-proc-generator/src/test/resources/expected/BidirectionalComputationStreamProc.java @@ -1,3 +1,22 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package org.neo4j.gds.beta.pregel.cc; import java.util.Map; From 4051289208373d78c360dcb4a740b5079f8c11ec Mon Sep 17 00:00:00 2001 From: Max Kiessling Date: Mon, 16 Jan 2023 15:48:46 +0100 Subject: [PATCH 017/400] Fix weird formatter choices and address review comments Co-authored-by: Martin Junghanns --- .../neo4j/gds/beta/pregel/PregelValidation.java | 4 ++-- .../neo4j/gds/pregel/proc/PregelBaseProc.java | 17 +++++------------ .../gds/pregel/proc/PregelBaseProcTest.java | 7 ++++--- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java index 5d218909d6e..da056385136 100644 --- a/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java +++ b/pregel-proc-generator/src/main/java/org/neo4j/gds/beta/pregel/PregelValidation.java @@ -75,7 +75,7 @@ final class PregelValidation { Optional validate(Element pregelElement) { if ( !isClass(pregelElement) || - !isPregelComputation(pregelElement) || + !isBasePregelComputation(pregelElement) || !isPregelProcedureConfig(pregelElement) || !hasEmptyConstructor(pregelElement) || !configHasFactoryMethod(pregelElement) @@ -123,7 +123,7 @@ private Optional pregelComputation(Element pregelElement, TypeMirr .findFirst(); } - private boolean isPregelComputation(Element pregelElement) { + private boolean isBasePregelComputation(Element pregelElement) { var pregelTypeElement = MoreElements.asType(pregelElement); var maybeInterface = pregelComputation(pregelElement, basePregelComputation); boolean isPregelComputation = maybeInterface.isPresent(); diff --git a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java index dd4dea0fa47..85cc326cd35 100644 --- a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java +++ b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java @@ -95,9 +95,7 @@ static void ensureInverseIndexesExist( }); } - static void ensureDirectedRelationships( - GraphStore graphStore, Collection relationshipTypes - ) { + static void ensureDirectedRelationships(GraphStore graphStore, Collection relationshipTypes) { var relationshipSchema = graphStore.schema().relationshipSchema(); var undirectedTypes = relationshipTypes .stream() @@ -137,22 +135,17 @@ static , CONFIG extends PregelConfig> List< nodePropertyValues = compositeNodeValue.doubleProperties(propertyKey).asNodeProperties(); break; case LONG_ARRAY: - nodePropertyValues = new HugeObjectArrayLongArrayPropertyValues(compositeNodeValue.longArrayProperties( - propertyKey)); + nodePropertyValues = new HugeObjectArrayLongArrayPropertyValues(compositeNodeValue.longArrayProperties(propertyKey)); break; case DOUBLE_ARRAY: - nodePropertyValues = new HugeObjectArrayDoubleArrayPropertyValues(compositeNodeValue.doubleArrayProperties( - propertyKey)); + nodePropertyValues = new HugeObjectArrayDoubleArrayPropertyValues(compositeNodeValue.doubleArrayProperties(propertyKey)); break; default: throw new IllegalArgumentException("Unsupported property type: " + element.propertyType()); } - return ImmutableNodeProperty.of(formatWithLocale("%s%s", propertyPrefix, propertyKey), - nodePropertyValues - ); - }) - .collect(Collectors.toList()); + return ImmutableNodeProperty.of(formatWithLocale("%s%s", propertyPrefix, propertyKey), nodePropertyValues); + }).collect(Collectors.toList()); } private PregelBaseProc() {} diff --git a/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java b/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java index 7de425bfd34..3fec11bbaf3 100644 --- a/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java +++ b/proc/pregel/src/test/java/org/neo4j/gds/pregel/proc/PregelBaseProcTest.java @@ -41,11 +41,12 @@ @GdlExtension class PregelBaseProcTest { - @GdlGraph @GdlGraph(graphNamePrefix = "undirected", orientation = Orientation.UNDIRECTED) - public static String GDL = "CREATE " + " ()-[:REL]->()," + " ()-[:REL2]->(),"; - + public static String GDL = + "CREATE " + + " ()-[:REL]->()," + + " ()-[:REL2]->(),"; @Inject GraphStore graphStore; From e6f4dcffc81b4d3f77b8c38ca2bc9b1976cae5ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 17 Jan 2023 10:44:29 +0100 Subject: [PATCH 018/400] Remove preview from 2.3 version --- doc/antora.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/antora.yml b/doc/antora.yml index dad9b547855..72675a5fc1b 100644 --- a/doc/antora.yml +++ b/doc/antora.yml @@ -1,6 +1,6 @@ name: graph-data-science title: Neo4j Graph Data Science -version: '2.3-preview' +version: '2.3' start_page: ROOT:index.adoc nav: - modules/ROOT/content-nav.adoc From 3a4170e396abf94919f1724afb695e7b0054fac2 Mon Sep 17 00:00:00 2001 From: Pontus Melke Date: Tue, 17 Jan 2023 08:42:08 +0100 Subject: [PATCH 019/400] Consolidate error handling Handle errors in the same way in NodeSubscriber and RelationshipSubscriber. --- .../core/loading/CypherRelationshipLoader.java | 3 +++ .../gds/core/loading/RelationshipSubscriber.java | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java index 4d6c3023807..432cbd0d587 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java @@ -129,6 +129,9 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { } subscriber.initialize(subscription.fieldNames(), propertyDefaultValueByName); CypherLoadingUtils.consume(subscription); + subscriber.error().ifPresent(e -> { + throw e; + }); progressTracker.endSubTask("Relationships"); return new BatchLoadResult(subscriber.rows(), -1L); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java index 2658ad2810c..fbdffd32e1c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipSubscriber.java @@ -25,11 +25,13 @@ import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.values.AnyValue; import org.neo4j.values.storable.NumberValue; import org.neo4j.values.storable.TextValue; +import java.util.Optional; import java.util.Set; import static org.neo4j.gds.RelationshipType.ALL_RELATIONSHIPS; @@ -65,6 +67,8 @@ class RelationshipSubscriber implements QuerySubscriber { private RelationshipsBuilder allRelationshipsBuilder; + private Optional error = Optional.empty(); + RelationshipSubscriber( IdMap idMap, CypherRelationshipLoader.Context loaderContext, @@ -105,6 +109,10 @@ void initialize(String[] fieldNames, ObjectDoubleMap propertyDefaultValu this.propertyValueBuffer = new double[propertyCount]; } + Optional error() { + return error; + } + public long rows() { return rows; } @@ -183,7 +191,13 @@ public void onRecordCompleted() { } @Override public void onError(Throwable throwable) { - throw new RuntimeException(throwable); + if (throwable instanceof RuntimeException) { + this.error = Optional.of((RuntimeException) throwable); + } else if (throwable instanceof QueryExecutionKernelException) { + this.error = Optional.of(((QueryExecutionKernelException) throwable).asUserException()); + } else { + this.error = Optional.of(new RuntimeException(throwable)); + } } @Override From 1fe1d39e60baa85f42727c9901fd5b36c78b7f94 Mon Sep 17 00:00:00 2001 From: Max Kiessling Date: Mon, 16 Jan 2023 17:37:23 +0100 Subject: [PATCH 020/400] Resolve conflict in PregelBaseProc after merge --- .../neo4j/gds/pregel/proc/PregelBaseProc.java | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java index 85cc326cd35..76f42acfff4 100644 --- a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java +++ b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java @@ -77,22 +77,30 @@ static void ensureInverseIndexesExist( Log log, TaskRegistryFactory taskRegistryFactory ) { - relationshipTypes + var relationshipTypesWithoutIndex = relationshipTypes .stream() .filter(relType -> !graphStore.inverseIndexedRelationshipTypes().contains(relType)) - .forEach(relType -> { - var inverseConfig = InverseRelationshipsConfigImpl - .builder() - .concurrency(concurrency) - .relationshipType(relType.name) - .build(); - - var inverseIndex = new InverseRelationshipsAlgorithmFactory() - .build(graphStore, inverseConfig, log, taskRegistryFactory) - .compute(); - - graphStore.addInverseIndex(relType, inverseIndex); - }); + .map(RelationshipType::name) + .collect(Collectors.toList()); + + if (relationshipTypesWithoutIndex.isEmpty()) { + return; + } + + var inverseConfig = InverseRelationshipsConfigImpl + .builder() + .concurrency(concurrency) + .relationshipTypes(relationshipTypesWithoutIndex) + .build(); + + new InverseRelationshipsAlgorithmFactory() + .build(graphStore, inverseConfig, log, taskRegistryFactory) + .compute() + .forEach((relationshipType, inverseIndex) -> graphStore.addInverseIndex( + relationshipType, + inverseIndex.topology(), + inverseIndex.properties() + )); } static void ensureDirectedRelationships(GraphStore graphStore, Collection relationshipTypes) { From d8fd9da0cfacc1f81060a0277620398c3cce2bba Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Tue, 17 Jan 2023 12:05:20 +0100 Subject: [PATCH 021/400] Set up RC version of Neo4j --- .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 196787de6ed..96c8575c08c 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -30,7 +30,8 @@ public enum Neo4jVersion { V_4_4, V_5_1, V_5_2, - V_5_3; + V_5_3, + V_RC; @Override public String toString() { @@ -43,6 +44,8 @@ public String toString() { return "5.2"; case V_5_3: return "5.3"; + case V_RC: + return "RC"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } @@ -115,6 +118,8 @@ static Neo4jVersion parse(String version) { return Neo4jVersion.V_5_2; } else if (minorVersion == 3) { return Neo4jVersion.V_5_3; + } else if (minorVersion == 4) { + return Neo4jVersion.V_RC; } } From c93aefdd120d0885afb9b017927f970fb2841d59 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Tue, 17 Jan 2023 13:13:26 +0100 Subject: [PATCH 022/400] RC is for 5.4 --- .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 96c8575c08c..c3f12691015 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -45,14 +45,14 @@ public String toString() { case V_5_3: return "5.3"; case V_RC: - return "RC"; + return "5.4"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } } public MajorMinorVersion semanticVersion() { - String version = toString(); + String version = toString(); var subVersions = version.split("\\."); if (subVersions.length < 2) { From 5158683b7dce163a7dd1e11713a002d3fd597f9c Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Tue, 17 Jan 2023 13:18:56 +0100 Subject: [PATCH 023/400] Support parsing 5.4 as RC --- .../src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index b7bb4fcdd2a..4368f4991b4 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -44,6 +44,7 @@ class Neo4jVersionTest { "5.2.0-dev, V_5_2", "5.2.0, V_5_2", "5.3.0, V_5_3", + "5.4.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); @@ -53,7 +54,6 @@ void testParse(String input, Neo4jVersion expected) { @CsvSource({ "dev", "4.3", // EOL - "5.4.0", // was dev at cutoff point "5.dev", "dev.5", "5.0", // 5.0 was never released to the public @@ -80,6 +80,7 @@ void shouldNotRespectVersionOverride() { "5.1.0, 5, 1", "5.2.0, 5, 2", "5.3.0, 5, 3", + "5.4.0, 5, 4", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); From 4b4574dfddf5f229bf4f3e4a332bbef558f261b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Mon, 16 Jan 2023 13:37:35 +0100 Subject: [PATCH 024/400] Rewrite fabric docs to composite database docs --- doc/modules/ROOT/content-nav.adoc | 2 +- .../{fabric.adoc => composite.adoc} | 85 ++++++++++++------- .../pages/production-deployment/index.adoc | 2 +- 3 files changed, 54 insertions(+), 35 deletions(-) rename doc/modules/ROOT/pages/production-deployment/{fabric.adoc => composite.adoc} (53%) diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index bb25da160f4..19fb6f4fb6e 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -146,7 +146,7 @@ ** xref:end-to-end-examples/fastrp-knn-example.adoc[] * xref:production-deployment/index.adoc[] ** xref:production-deployment/transaction-handling.adoc[] -** xref:production-deployment/fabric.adoc[] +** xref:production-deployment/composite.adoc[] ** xref:production-deployment/neo4j-cluster.adoc[] ** xref:production-deployment/feature-toggles.adoc[] * xref:python-client/index.adoc[] diff --git a/doc/modules/ROOT/pages/production-deployment/fabric.adoc b/doc/modules/ROOT/pages/production-deployment/composite.adoc similarity index 53% rename from doc/modules/ROOT/pages/production-deployment/fabric.adoc rename to doc/modules/ROOT/pages/production-deployment/composite.adoc index 335dddc827b..3fc171774b0 100644 --- a/doc/modules/ROOT/pages/production-deployment/fabric.adoc +++ b/doc/modules/ROOT/pages/production-deployment/composite.adoc @@ -1,41 +1,60 @@ -[[fabric]] -= Using GDS and Fabric -:description: This section describes how the Neo4j Graph Data Science library can be used in a Neo4j Fabric deployment. +[[composite]] +// Putting "Fabric" in the header might help with searching for the +// case that users are not familiar with composite databases yet += Using GDS and composite databases (formerly known as Fabric) +:description: This section describes how the Neo4j Graph Data Science library can be used in a Neo4j composite database deployment. include::partial$/common-usage/not-on-aurads-note.adoc[] -Neo4j Fabric is a way to store and retrieve data in multiple databases, whether they are on the same Neo4j DBMS or in multiple DBMSs, using a single Cypher query. -For more information about Fabric itself, please visit the https://neo4j.com/docs/operations-manual/4.4/fabric/introduction/[Fabric documentation]. +Neo4j composite databases are a way to store and retrieve data in multiple databases, whether they are on the same Neo4j DBMS or in multiple DBMSs, using a single Cypher query. +For more information about Composite databases/Fabric itself, please visit the -A typical Neo4j Fabric setup consists of two components: one or more shards that hold the data and one or more Fabric proxies that coordinate the distributed queries. -There are two ways of running the Neo4j Graph Data Science library in a Fabric deployment, both of which are covered in this section: +[.tabbed-example, caption = ] +==== - . Running GDS on a Fabric <> - . Running GDS on a Fabric <> +[.include-with-neo4j-4.x] +===== +https://neo4j.com/docs/operations-manual/4.4/fabric/introduction/[Fabric documentation]. +===== -[[fabric-shard]] +[.include-with-neo4j-5.x] +===== +https://neo4j.com/docs/operations-manual/current/composite-databases/[Composite databases documentation]. +===== + +==== + +NOTE: For simplicity this documentation page further only mentions composite databases which are available from Neo4j 5.0 on. As GDS supports 4.x and 5.x Neo4j versions this documentation can be also applied to Fabric setups using the exact same queries and examples as shown below. + +A typical Neo4j composite setup consists of two components: one or more shards (constituents) that hold the data and one composite database that coordinates the distributed queries. +There are two ways of running the Neo4j Graph Data Science library in a composite deployment, both of which are covered in this section: + + . Running GDS on a Composite <> + . Running GDS on a Composite <> + +[[composite-shard]] == Running GDS on the Shards -In this mode of using GDS in a Fabric environment, the GDS operations are executed on the shards. -The graph projections and algorithms are then executed on each shard individually, and the results can be combined via the Fabric proxy. +In this mode of using GDS in a composite environment, the GDS operations are executed on the shards. +The graph projections and algorithms are then executed on each shard individually, and the results can be combined via the composite database. This scenario is useful, if the graph is partitioned into disjoint subgraphs across shards, i.e. there is no logical relationship between nodes on different shards. Another use case is to replicate the graph's topology across multiple shards, where some shards act as operational and others as analytical databases. -[[fabric-shard-setup]] +[[composite-shard-setup]] === Setup In this scenario we need to set up the shards to run the Neo4j Graph Data Science library. Every shard that will run the Graph Data Science library should be configured just as a standalone GDS database would be, for more information see xref:installation/index.adoc[Installation]. -The Fabric proxy nodes do not require any special configuration, i.e., the GDS library plugin does not need to be installed. -However, the proxy nodes should be configured to handle the amount of data received from the shards. +The composite database does not require any special configuration, i.e., the GDS library plugin does not need to be installed. +However, the Composite database should be configured to handle the amount of data received from the shards. -[[fabric-shard-examples]] +[[composite-shard-examples]] === Examples -Let's assume we have a Fabric setup with two shards. +Let's assume we have a composite setup with two shards. One shard functions as the operational database and holds a graph with the schema `(Person)-[KNOWS]->(Person)`. Every `Person` node also stores an identifying property `id` and the persons `name` and possibly other properties. @@ -46,7 +65,7 @@ First we need to project a named graph on the analytical database shard. [source, cypher, role=noplay] ---- CALL { - USE FABRIC_DB_NAME.ANALYTICS_DB + USE COMPOSITE_DB_NAME.ANALYTICS_DB CALL gds.graph.project('graph', 'Person', 'KNOWS') YIELD graphName RETURN graphName @@ -54,18 +73,18 @@ CALL { RETURN graphName ---- -Using Fabric, we can now calculate the PageRank score for each Person and join the results with the name of that Person. +Using the composite database, we can now calculate the PageRank score for each Person and join the results with the name of that Person. [source, cypher, role=noplay] ---- CALL { - USE FABRIC_DB_NAME.ANALYTICS_DB + USE COMPOSITE_DB_NAME.ANALYTICS_DB CALL gds.pagerank.stream('graph', {}) YIELD nodeId, score AS pageRank RETURN gds.util.asNode(nodeId).id AS personId, pageRank } CALL { - USE FABRIC_DB_NAME.OPERATIONAL_DB + USE COMPOSITE_DB_NAME.OPERATIONAL_DB WITH personId MATCH (p {id: personId}) RETURN p.name AS name @@ -78,32 +97,32 @@ The algorithm results are streamed to the proxy, together with the unique node i For every row returned by the first subquery, the operational database is then queried for the persons name, again using the unique node id to identify the `Person` node across the shards. -[[fabric-proxy]] -== Running GDS on the Fabric Proxy +[[composite-proxy]] +== Running GDS on the Composite database -In this mode of using GDS in a Fabric environment, the GDS operations are executed on the Fabric proxy server. +In this mode of using GDS in a composite environment, the GDS operations are executed on the Fabric proxy server. The graph projections are then using the data stored on the shards to construct the in-memory graph. -NOTE: Currently only xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] is supported for projecting in-memory graphs on a Fabric proxy. +NOTE: Currently only xref:management-ops/projections/graph-project-cypher-aggregation.adoc[Cypher Aggregation] is supported for projecting in-memory graphs on a Composite database. -Graph algorithms can then be executed on the Fabric proxy, similar to a single machine setup. -This scenario is useful, if a graph, that logically represents a single graph, is distributed to different Fabric shards. +Graph algorithms can then be executed on the composite database, similar to a single machine setup. +This scenario is useful, if a graph that logically represents a single graph is distributed to different Composite shards. -[[fabric-proxy-setup]] +[[composite-proxy-setup]] === Setup In this scenario we need to set up the proxy to run the Neo4j Graph Data Science library. -The dbms that manages the Fabric proxy database needs to have the GDS plugin installed and configured. +The dbms that manages the composite database needs to have the GDS plugin installed and configured. For more information see xref:installation/index.adoc[Installation]. The proxy node should also be configured to handle the amount of data received from the shards as well as executing graph projections and algorithms. Fabric shards do not need any special configuration, i.e., the GDS library plugin does not need to be installed. -[[fabric-proxy-examples]] +[[composite-proxy-examples]] === Examples -Let's assume we have a Fabric setup with two shards. +Let's assume we have a composite setup with two shards. Both shards function as the operational databases and hold graphs with the schema `(Person)-[KNOWS]->(Person)`. We now need to query the shards in order to drive the import process on the proxy node. @@ -111,11 +130,11 @@ We now need to query the shards in order to drive the import process on the prox [source, cypher, role=noplay] ---- CALL { - USE FABRIC_DB_NAME.FABRIC_SHARD_0_NAME + USE COMPOSITE_DB_NAME.COMPOSITE_SHARD_0_NAME MATCH (p:Person) OPTIONAL MATCH (p)-[:KNOWS]->(n:Person) RETURN p, n UNION - USE FABRIC_DB_NAME.FABRIC_SHARD_1_NAME + USE COMPOSITE_DB_NAME.COMPOSITE_SHARD_1_NAME MATCH (p:Person) OPTIONAL MATCH (p)-[:KNOWS]->(n:Person) RETURN p, n } diff --git a/doc/modules/ROOT/pages/production-deployment/index.adoc b/doc/modules/ROOT/pages/production-deployment/index.adoc index d60b2a87270..44959c780b0 100644 --- a/doc/modules/ROOT/pages/production-deployment/index.adoc +++ b/doc/modules/ROOT/pages/production-deployment/index.adoc @@ -6,6 +6,6 @@ This chapter is divided into the following sections: * xref:production-deployment/transaction-handling.adoc[Transaction Handling] -* xref:production-deployment/fabric.adoc[Using GDS and Fabric] +* xref:production-deployment/composite.adoc[Using GDS and Composite databases] * xref:production-deployment/neo4j-cluster.adoc[GDS with Neo4j cluster] * xref:production-deployment/feature-toggles.adoc[GDS Feature Toggles] From 11598634dc809a0a9ced204c780c28dd683070be Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 17 Jan 2023 15:22:35 +0000 Subject: [PATCH 025/400] Update the sampled WCC example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Martin Junghanns Co-authored-by: Florentin Dörre --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index decc461ec42..36fb24093ce 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -631,7 +631,7 @@ This time, we execute WCC on `myIndexedGraph` which will allow the algorithm to .The following will run the algorithm with sampled strategy and stream results: [source, cypher, role=noplay] ---- -CALL gds.wcc.stream('myIndexedGraph') +CALL gds.wcc.stream('myIndexedGraph', { consecutiveIds: true }) YIELD nodeId, componentId RETURN gds.util.asNode(nodeId).name AS name, componentId ORDER BY componentId, name @@ -644,11 +644,11 @@ ORDER BY componentId, name | "Alice" | 0 | "Bridget" | 0 | "Charles" | 0 -| "Doug" | 3 -| "Mark" | 3 -| "Michael" | 3 +| "Doug" | 1 +| "Mark" | 1 +| "Michael" | 1 |=== -- -As expected, the results are the same as before. +Here we also demonstrate the use of `conscutiveIds: true`, the results are semantically the same as before. However, on larger graphs, there will be a notable improvement in compute time. From 5219de338e432f5d54621f50a95e52f786d3264f Mon Sep 17 00:00:00 2001 From: Max Kiessling Date: Mon, 16 Jan 2023 19:35:34 +0100 Subject: [PATCH 026/400] Add documentation for bidirectional pregel computation --- .../ROOT/pages/algorithms/pregel-api.adoc | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc index be357ed0537..c368009bf55 100644 --- a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc +++ b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc @@ -404,6 +404,45 @@ public class CustomComputation implements PregelComputation { } ---- +[[algorithms-pregel-api-bidirectional]] +=== Traversing incoming relationships + +Some algorithms implemented in Pregel might require or benefit from the ability to access and send messages to all incoming relationships of the current context node. +GDS supports the creation of inverse indexes for relationship types, which enables the traversal of incoming relationships for directed relationship types. + +A Pregel algorithm can access this index by implementing the `org.neo4j.gds.beta.pregel.BidirectionalPregelComputation` interface instead of the `PregelComputation` interface. +Implementing this interface has the following consequences: + +* The Pregel framework will make sure that all relationships passed into the algorithm are inverse indexed. + If no such index exists, an error will be thrown. +* The signature of the `init` and `compute` functions now accept a `org.neo4j.gds.beta.pregel.context.InitContext.BidirectionalInitContext` and `org.neo4j.gds.beta.pregel.context.ComputeContext.BidirectionalComputeContext` respectively. +* Algorithms annotated with the `@PregelProcedure` annotation will automatically create all required inverse indexes. + +The `BidirectionalInitContext` and `BidirectionalComputeContexts` expose the following new methods in addition to the methods defined by `InitContext` and `ComputeContext`: + +[source, java] +---- +//Returns the incoming degree (number of relationships) of the currently processed node. +public int incomingDegree(); +// Calls the consumer for each incoming neighbor of the currently processed node. +public void forEachIncomingNeighbor(LongConsumer targetConsumer); +// Calls the consumer for each incoming neighbor of the given node. +public void forEachIncomingNeighbor(long nodeId, LongConsumer targetConsumer); +// Calls the consumer once for each incoming neighbor of the currently processed node. +public void forEachDistinctIncomingNeighbor(LongConsumer targetConsumer); +// Calls the consumer once for each incoming neighbor of the given node. +public void forEachDistinctIncomingNeighbor(long nodeId, LongConsumer targetConsumer); +---- + +In addition, the `BidirectionalComputeContext` also exposes the following function: + +[source, java] +---- +// Sends the given message to all neighbors of the node. +public void sendToIncomingNeighbors(double message); +---- + + [[algorithms-pregel-api-logging]] === Logging From c5dba4eed1930227dc76681879481c978695bbfb Mon Sep 17 00:00:00 2001 From: Max Kiessling Date: Tue, 17 Jan 2023 22:33:56 +0100 Subject: [PATCH 027/400] Make sure that relationship type index is created when using PBI - create an integration test for arrow database import --- .../gds/core/io/db/GraphStoreToDatabaseExporterConfig.java | 2 +- .../gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfig.java index 7282a6d3b38..da7b0912b6d 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfig.java @@ -107,7 +107,7 @@ default org.neo4j.internal.batchimport.Configuration toBatchImporterConfig() { exportConfig.writeConcurrency(), exportConfig.pageCacheMemory(), exportConfig.highIO(), - IndexConfig.DEFAULT.withLabelIndex() + IndexConfig.DEFAULT.withLabelIndex().withRelationshipTypeIndex() ); } } diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java index a660836167b..d0f132f6dad 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java @@ -43,6 +43,8 @@ void testToBatchImporterConfig() { assertThat(pbiConfig.pageCacheMemory()).isEqualTo(100_000L); assertThat(pbiConfig.highIO()).isTrue(); assertThat(Neo4jProxy.writeConcurrency(pbiConfig)).isEqualTo(42); + assertThat(pbiConfig.indexConfig().createLabelIndex()).isTrue(); + assertThat(pbiConfig.indexConfig().createRelationshipIndex()).isTrue(); } } From d1e9d3a19232a4161c5e33b663c18957d0f0f1b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 18 Jan 2023 11:09:07 +0100 Subject: [PATCH 028/400] Fix test assertion to include RC compat Co-authored-by: Adam Schill Collberg --- .../src/test/java/org/neo4j/gds/SysInfoProcTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index d04f83f734e..a53e7dc68c7 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -150,6 +150,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.3" ); break; + case V_RC: + expectedCompatibilities = Set.of( + "Neo4j Settings RC (placeholder)", + "Neo4j Settings RC", + "Neo4j RC (placeholder)", + "Neo4j RC" + ); + break; default: throw new IllegalStateException("Unexpected Neo4j version: " + neo4jVersion); } From 0920c7640d32f9034234265b5221913dc7900aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 18 Jan 2023 11:30:16 +0100 Subject: [PATCH 029/400] Ignore the result in the sampled WCC example The component ids are not deterministic, so for now we just ignore them Co-authored-by: Adam Schill Collberg --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index 36fb24093ce..c320247adc6 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -626,12 +626,12 @@ CALL gds.graph.project( The following query is identical to the stream example in the xref:algorithms/wcc.adoc#algorithms-wcc-examples-seeding[previous section]. This time, we execute WCC on `myIndexedGraph` which will allow the algorithm to use the sampled strategy. -[role=query-example] +[role=query-example, no-result=true] -- .The following will run the algorithm with sampled strategy and stream results: [source, cypher, role=noplay] ---- -CALL gds.wcc.stream('myIndexedGraph', { consecutiveIds: true }) +CALL gds.wcc.stream('myIndexedGraph') YIELD nodeId, componentId RETURN gds.util.asNode(nodeId).name AS name, componentId ORDER BY componentId, name @@ -644,11 +644,14 @@ ORDER BY componentId, name | "Alice" | 0 | "Bridget" | 0 | "Charles" | 0 -| "Doug" | 1 -| "Mark" | 1 -| "Michael" | 1 +| "Doug" | 3 +| "Mark" | 3 +| "Michael" | 3 |=== -- -Here we also demonstrate the use of `conscutiveIds: true`, the results are semantically the same as before. -However, on larger graphs, there will be a notable improvement in compute time. +[NOTE] +==== +Because of the randomness in the Graph sampling optimization, running it another time might yield a different componentIds. +For our case here it would be equally plausible to get the inverse solution, f.i. when our community `0` nodes are mapped to community `3` instead, and vice versa. +==== From 15d2adb0381ea795cf1915e735825492200750b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 17 Jan 2023 16:07:34 +0100 Subject: [PATCH 030/400] Add keywords for api tiers This helps to find the explanation of the api tiers --- doc/modules/ROOT/pages/introduction.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/introduction.adoc b/doc/modules/ROOT/pages/introduction.adoc index 74aedcb177b..f59bcf7a26a 100644 --- a/doc/modules/ROOT/pages/introduction.adoc +++ b/doc/modules/ROOT/pages/introduction.adoc @@ -1,7 +1,7 @@ [[introduction]] = Introduction :description: This chapter provides a brief introduction of the main concepts in the Neo4j Graph Data Science library. - +:keywords: alpha, beta, Production-quality, api tiers The Neo4j Graph Data Science (GDS) library provides efficiently implemented, parallel versions of common graph algorithms, exposed as Cypher procedures. Additionally, GDS includes machine learning pipelines to train predictive supervised models to solve graph problems, such as predicting missing relationships. From 2b8e8140804c92dc30fd6459db9824abd636baab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 18 Jan 2023 11:59:50 +0100 Subject: [PATCH 031/400] Map supported versions from Neo4j -> GDS Cherry-picked #6835 --- .../supported-neo4j-versions.adoc | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 562ac0194ea..3f78436afce 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -4,22 +4,15 @@ Below is the compatibility matrix for the GDS library vs Neo4j. In general, you can count on the latest version of GDS supporting the latest version of Neo4j and vice versa, and we recommend you always upgrade to that combination. -We list software with major and minor version only, e.g. GDS library 1.8. -You should read that as any patch version of that major+minor version, but again, do upgrade to the latest patch always, to ensure you get all bug fixes included. - Not finding your version of GDS or Neo4j listed? Time to upgrade! [opts=header] |=== -| Neo4j Graph Data Science | Neo4j version -.3+<.^|`2.3` -| `5.1` -| `5.2` -| `4.4`, at least `4.4.9` -.4+<.^|`2.2` -| `5.1` -| `5.2` -| `4.4`, at least `4.4.9` -| `4.3`, at least `4.3.15` +| Neo4j version | Neo4j Graph Data Science +| `5.3` | `2.3`, `2.2.6 or later` +| `5.2` | `2.3`, `2.2.3 or later` +| `5.1`| `2.3`, `2.2.1` +| `4.4.9 or later`| `2.3`, `2.2` +| `4.3.15 or later` | `2.2` |=== From 972574d349f91016076ecaa908383cf5d824a159 Mon Sep 17 00:00:00 2001 From: Brian Shi Date: Wed, 18 Jan 2023 13:56:06 +0000 Subject: [PATCH 032/400] Add bloom link to content-nav.adoc --- doc/modules/ROOT/content-nav.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 19fb6f4fb6e..66be7da9f18 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -150,6 +150,7 @@ ** xref:production-deployment/neo4j-cluster.adoc[] ** xref:production-deployment/feature-toggles.adoc[] * xref:python-client/index.adoc[] +* link:https://neo4j.com/docs/bloom-user-guide/current/bloom-tutorial/gds-integration/[Bloom visualization] * Appendix ** xref:operations-reference/appendix-a.adoc[] *** xref:operations-reference/graph-operation-references.adoc[] From 7309eb064a8b9ee3adc736ded42750653979d069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 19 Jan 2023 12:10:04 +0100 Subject: [PATCH 033/400] Upgrade pregel bootstrap shadow jar plugin to latest version - compatibility with Java 17 - update GDS dependency to 2.3.0 --- examples/pregel-bootstrap/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index 2eccbd2a850..e5e2a3aa3f0 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -2,12 +2,12 @@ plugins { // Apply the java plugin to add support for Java id 'java' // Used for building a standalone jar - id 'com.github.johnrengelman.shadow' version '4.0.4' + id 'com.github.johnrengelman.shadow' version '7.1.2' } ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.0-alpha05' + gdsVersion = '2.3.0' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs From 9ddb06cf62963f3ca2dbe3455e498c08bf415421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 11 Jan 2023 13:51:11 +0100 Subject: [PATCH 034/400] Add tests to show two problems with KSpanningTree Co-Authored-By: Ioannis Panagiotas --- .../neo4j/gds/spanningtree/SpanningTree.java | 11 ++- .../impl/spanningtree/KSpanningTreeTest.java | 79 +++++++++++++++++++ 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java index 47781805f74..091815c1bf0 100644 --- a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java +++ b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java @@ -20,7 +20,7 @@ package org.neo4j.gds.spanningtree; import org.apache.commons.lang3.builder.EqualsBuilder; -import org.neo4j.gds.api.RelationshipConsumer; +import org.neo4j.gds.api.RelationshipWithPropertyConsumer; import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -62,7 +62,9 @@ public double totalWeight() { return totalWeight; } - public HugeLongArray parentArray() {return parent;} + public HugeLongArray parentArray() { + return parent; + } public long parent(long nodeId) {return parent.get(nodeId);} @@ -70,13 +72,14 @@ public double costToParent(long nodeId) { return costToParent.get(nodeId); } - public void forEach(RelationshipConsumer consumer) { + public void forEach(RelationshipWithPropertyConsumer consumer) { for (int i = 0; i < nodeCount; i++) { final long parent = this.parent.get(i); + final double cost = this.costToParent(i); if (parent == -1) { continue; } - if (!consumer.accept(parent, i)) { + if (!consumer.accept(parent, i, cost)) { return; } } diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java index 4b1c2abee89..30227053f9c 100644 --- a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.impl.spanningtree; +import org.apache.commons.lang3.mutable.MutableLong; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -29,9 +30,13 @@ import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.gdl.GdlFactory; import org.neo4j.gds.spanningtree.Prim; import org.neo4j.gds.spanningtree.SpanningTree; +import java.util.HashSet; + +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -124,4 +129,78 @@ void testNeoIdsWithOffset() { assertEquals(spanningTree, otherSpanningTree); } + + @Test + void shouldProduceSingleConnectedTree() { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (a)-[:TYPE {cost: 1.0}]->(b)" + + ", (b)-[:TYPE {cost: 20.0}]->(c)" + + ", (c)-[:TYPE {cost: 30.0}]->(d)" + + ", (d)-[:TYPE {cost: 1.0}]->(e)" + ); + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("a"); + + var k = 3; + var spanningTree = new KSpanningTree( + graph, + Prim.MIN_OPERATOR, + startNode, + k, + ProgressTracker.NULL_TRACKER + ).compute(); + + // if there are more than k nodes then there is more than one root + // meaning there is more than one tree (or the tree is broken) + var nodesInTree = new HashSet(); + spanningTree.forEach((s, t, __) -> { + nodesInTree.add(s); + nodesInTree.add(t); + return true; + }); + + assertThat(nodesInTree.size()).isEqualTo(k); + } + + @Test + void shouldProduceSingleTreeWithKMinusOneEdges() { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (f:Node)" + + ", (a)-[:TYPE {cost: 1.0}]->(b)" + + ", (b)-[:TYPE {cost: 20.0}]->(c)" + + ", (c)-[:TYPE {cost: 30.0}]->(d)" + + ", (d)-[:TYPE {cost: 1.0}]->(e)" + + ", (e)-[:TYPE {cost: 1.0}]->(f)" + ); + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("a"); + + var k = 2; + + var spanningTree = new KSpanningTree( + graph, + Prim.MIN_OPERATOR, + startNode, + 2, + ProgressTracker.NULL_TRACKER + ).compute(); + + var counter = new MutableLong(0); + spanningTree.forEach((__, ___, ____) -> { + counter.add(1); + return true; + }); + + assertThat(counter.getValue()).isEqualTo(k - 1); + } } From fce9e46f2f8eb4db726c35ef4048c0c9532edd04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 11 Jan 2023 14:28:31 +0100 Subject: [PATCH 035/400] Clean up and enable test Co-Authored-By: Ioannis Panagiotas --- .../impl/spanningtree/KSpanningTreeTest.java | 69 ++++++++----------- 1 file changed, 29 insertions(+), 40 deletions(-) diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java index 30227053f9c..a2a712a2300 100644 --- a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java @@ -21,7 +21,6 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; @@ -32,20 +31,17 @@ import org.neo4j.gds.extension.Inject; import org.neo4j.gds.gdl.GdlFactory; import org.neo4j.gds.spanningtree.Prim; -import org.neo4j.gds.spanningtree.SpanningTree; import java.util.HashSet; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; /** - * 1 - * (x) >(a)---(d) (x) (a) (d) - * /3 \2 /3 => / / - * (b)---(c) (b) (c) - * 1 + * 1 + * (x), (a)---(d) (x) (a) (d) + * /3 \2 /3 => / / + * (b)---(c) (b) (c) + * 1 */ @GdlExtension class KSpanningTreeTest { @@ -71,6 +67,14 @@ class KSpanningTreeTest { @Inject private IdFunction idFunction; + private static final int OFFSET = 5; + + @GdlGraph(idOffset = OFFSET, orientation = Orientation.UNDIRECTED, graphNamePrefix = "offset") + private static final String DB_CYPHER_WITH_OFFSET = DB_CYPHER; + + @Inject + private Graph offsetGraph; + private int a, b, c, d, x; @BeforeEach @@ -84,50 +88,35 @@ void setUp() { @Test void testMaximumKSpanningTree() { - final SpanningTree spanningTree = new KSpanningTree( - graph, - Prim.MAX_OPERATOR, - a, - 2, - ProgressTracker.NULL_TRACKER - ) + var spanningTree = new KSpanningTree(graph, Prim.MAX_OPERATOR, a, 2, ProgressTracker.NULL_TRACKER) .compute(); - assertEquals(spanningTree.head(a), spanningTree.head(b)); - assertEquals(spanningTree.head(c), spanningTree.head(d)); - assertNotEquals(spanningTree.head(a), spanningTree.head(c)); - assertNotEquals(spanningTree.head(a), spanningTree.head(x)); - assertNotEquals(spanningTree.head(c), spanningTree.head(x)); + assertThat(spanningTree.head(a)).isEqualTo(spanningTree.head(b)); + assertThat(spanningTree.head(c)).isEqualTo(spanningTree.head(d)); + assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(c)); + assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(x)); + assertThat(spanningTree.head(c)).isNotEqualTo(spanningTree.head(x)); } @Test void testMinimumKSpanningTree() { - final SpanningTree spanningTree = new KSpanningTree( - graph, - Prim.MIN_OPERATOR, - a, - 2, - ProgressTracker.NULL_TRACKER - ) + var spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, a, 2, ProgressTracker.NULL_TRACKER) .compute(); - assertEquals(spanningTree.head(a), spanningTree.head(d)); - assertEquals(spanningTree.head(b), spanningTree.head(c)); - assertNotEquals(spanningTree.head(a), spanningTree.head(b)); - assertNotEquals(spanningTree.head(a), spanningTree.head(x)); - assertNotEquals(spanningTree.head(b), spanningTree.head(x)); + assertThat(spanningTree.head(a)).isEqualTo(spanningTree.head(d)); + assertThat(spanningTree.head(b)).isEqualTo(spanningTree.head(c)); + assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(b)); + assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(x)); + assertThat(spanningTree.head(b)).isNotEqualTo(spanningTree.head(x)); } @Test - @Disabled("Need to extend GdlGraph to generate offset node IDs and fix the test") void testNeoIdsWithOffset() { - SpanningTree spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, 0, 2, ProgressTracker.NULL_TRACKER) - .compute(); - - SpanningTree otherSpanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, 5, 2, ProgressTracker.NULL_TRACKER) - .compute(); + var spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, 0, 2, ProgressTracker.NULL_TRACKER).compute(); + var otherSpanningTree = new KSpanningTree(offsetGraph, Prim.MIN_OPERATOR, OFFSET, 2, ProgressTracker.NULL_TRACKER).compute(); - assertEquals(spanningTree, otherSpanningTree); + assertThat(spanningTree.parentArray().toArray()) + .containsExactly(otherSpanningTree.parentArray().toArray()); } @Test From 930c7d2dabeb7a6a3391b993754e045b5a1effac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 11 Jan 2023 14:57:35 +0100 Subject: [PATCH 036/400] Remove the right amount of relationships Co-Authored-By: Ioannis Panagiotas --- .../org/neo4j/gds/impl/spanningtree/KSpanningTree.java | 9 +++++---- .../neo4j/gds/impl/spanningtree/KSpanningTreeTest.java | 6 ++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index b74a506f5f9..ce5ca6781aa 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -75,8 +75,9 @@ public SpanningTree compute() { SpanningTree spanningTree = prim.compute(); HugeLongArray parent = spanningTree.parentArray(); long parentSize = parent.size(); - HugeLongPriorityQueue priorityQueue = minMax == Prim.MAX_OPERATOR ? HugeLongPriorityQueue.min(parentSize) : HugeLongPriorityQueue.max( - parentSize); + HugeLongPriorityQueue priorityQueue = minMax == Prim.MAX_OPERATOR + ? HugeLongPriorityQueue.min(parentSize) + : HugeLongPriorityQueue.max(parentSize); progressTracker.beginSubTask(parentSize); for (long i = 0; i < parentSize && terminationFlag.running(); i++) { long p = parent.get(i); @@ -88,8 +89,8 @@ public SpanningTree compute() { } progressTracker.endSubTask(); progressTracker.beginSubTask(k - 1); - // remove k-1 relationships - for (long i = 0; i < k - 1 && terminationFlag.running(); i++) { + // remove until there are k-1 relationships + for (long i = 0; i < spanningTree.effectiveNodeCount() - k && terminationFlag.running(); i++) { long cutNode = priorityQueue.pop(); parent.set(cutNode, -1); progressTracker.logProgress(); diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java index a2a712a2300..46250e1142c 100644 --- a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java @@ -91,8 +91,7 @@ void testMaximumKSpanningTree() { var spanningTree = new KSpanningTree(graph, Prim.MAX_OPERATOR, a, 2, ProgressTracker.NULL_TRACKER) .compute(); - assertThat(spanningTree.head(a)).isEqualTo(spanningTree.head(b)); - assertThat(spanningTree.head(c)).isEqualTo(spanningTree.head(d)); + assertThat(spanningTree).matches(tree -> tree.head(a) == tree.head(b) ^ tree.head(c) == tree.head(d)); assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(c)); assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(x)); assertThat(spanningTree.head(c)).isNotEqualTo(spanningTree.head(x)); @@ -103,8 +102,7 @@ void testMinimumKSpanningTree() { var spanningTree = new KSpanningTree(graph, Prim.MIN_OPERATOR, a, 2, ProgressTracker.NULL_TRACKER) .compute(); - assertThat(spanningTree.head(a)).isEqualTo(spanningTree.head(d)); - assertThat(spanningTree.head(b)).isEqualTo(spanningTree.head(c)); + assertThat(spanningTree).matches(tree -> tree.head(a) == tree.head(d) ^ tree.head(b) == tree.head(c)); assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(b)); assertThat(spanningTree.head(a)).isNotEqualTo(spanningTree.head(x)); assertThat(spanningTree.head(b)).isNotEqualTo(spanningTree.head(x)); From 61e66b17619b62222e508a4712520067a7be7162 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 12 Jan 2023 09:52:55 +0100 Subject: [PATCH 037/400] Fix proc test --- .../KSpanningTreeWriteProcTest.java | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java index d343b89dab0..89095f9143a 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java @@ -31,16 +31,14 @@ import java.util.HashMap; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; class KSpanningTreeWriteProcTest extends BaseProcTest { - private static String GRAPH_NAME = "graph"; + private static final String GRAPH_NAME = "graph"; @Neo4jGraph - static final String DB_CYPHER = + private static final String DB_CYPHER = "CREATE (a:Node {name:'a'})\n" + "CREATE (b:Node {name:'b'})\n" + "CREATE (c:Node {name:'c'})\n" + @@ -54,7 +52,7 @@ class KSpanningTreeWriteProcTest extends BaseProcTest { " (d)-[:TYPE {w:3.0}]->(c)"; @Inject - IdFunction idFunction; + private IdFunction idFunction; @BeforeEach void setupGraph() throws Exception { @@ -80,9 +78,9 @@ void testMax() { .yields("preProcessingMillis", "computeMillis", "writeMillis"); runQueryWithRowConsumer(query, row -> { - assertTrue(row.getNumber("preProcessingMillis").longValue() >= 0); - assertTrue(row.getNumber("writeMillis").longValue() >= 0); - assertTrue(row.getNumber("computeMillis").longValue() >= 0); + assertThat(row.getNumber("preProcessingMillis").longValue() >= 0).isTrue(); + assertThat(row.getNumber("writeMillis").longValue() >= 0).isTrue(); + assertThat(row.getNumber("computeMillis").longValue() >= 0).isTrue(); }); final HashMap communities = new HashMap<>(); @@ -93,9 +91,8 @@ void testMax() { communities.put(name, p); }); - assertEquals(communities.get("a"), communities.get("b")); - assertEquals(communities.get("d"), communities.get("c")); - assertNotEquals(communities.get("a"), communities.get("c")); + assertThat(communities).matches(c -> c.get("a").equals(c.get("b")) ^ c.get("c").equals(c.get("d"))); + assertThat(communities.get("a")).isNotEqualTo(communities.get("c")); } @Test @@ -109,11 +106,10 @@ void testMin() { .addParameter("writeProperty", "partition") .yields("preProcessingMillis", "computeMillis", "writeMillis"); - runQueryWithRowConsumer(query, row -> { - assertTrue(row.getNumber("preProcessingMillis").longValue() >= 0); - assertTrue(row.getNumber("writeMillis").longValue() >= 0); - assertTrue(row.getNumber("computeMillis").longValue() >= 0); + assertThat(row.getNumber("preProcessingMillis").longValue() >= 0).isTrue(); + assertThat(row.getNumber("writeMillis").longValue() >= 0).isTrue(); + assertThat(row.getNumber("computeMillis").longValue() >= 0).isTrue(); }); final HashMap communities = new HashMap<>(); @@ -124,9 +120,7 @@ void testMin() { communities.put(name, p); }); - assertEquals(communities.get("a"), communities.get("d")); - assertEquals(communities.get("b"), communities.get("c")); - assertNotEquals(communities.get("a"), communities.get("b")); + assertThat(communities).matches(c -> c.get("a").equals(c.get("d")) ^ c.get("b").equals(c.get("c"))); + assertThat(communities.get("a")).isNotEqualTo(communities.get("b")); } - } From 612e5a8b83451535d6da7f26f23d20fbf94318cd Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 16 Jan 2023 09:43:26 +0100 Subject: [PATCH 038/400] Minor refactoring --- .../gds/impl/spanningtree/KSpanningTree.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index ce5ca6781aa..23105bdaac8 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.impl.spanningtree; +import org.jetbrains.annotations.NotNull; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.utils.paged.HugeLongArray; @@ -71,13 +72,12 @@ public SpanningTree compute() { startNodeId, progressTracker ); + prim.setTerminationFlag(getTerminationFlag()); SpanningTree spanningTree = prim.compute(); HugeLongArray parent = spanningTree.parentArray(); long parentSize = parent.size(); - HugeLongPriorityQueue priorityQueue = minMax == Prim.MAX_OPERATOR - ? HugeLongPriorityQueue.min(parentSize) - : HugeLongPriorityQueue.max(parentSize); + HugeLongPriorityQueue priorityQueue = createPriorityQueue(parentSize); progressTracker.beginSubTask(parentSize); for (long i = 0; i < parentSize && terminationFlag.running(); i++) { long p = parent.get(i); @@ -90,7 +90,8 @@ public SpanningTree compute() { progressTracker.endSubTask(); progressTracker.beginSubTask(k - 1); // remove until there are k-1 relationships - for (long i = 0; i < spanningTree.effectiveNodeCount() - k && terminationFlag.running(); i++) { + long numberOfDeletions = spanningTree.effectiveNodeCount() - k; + for (long i = 0; i < numberOfDeletions && terminationFlag.running(); i++) { long cutNode = priorityQueue.pop(); parent.set(cutNode, -1); progressTracker.logProgress(); @@ -101,6 +102,14 @@ public SpanningTree compute() { return this.spanningTree; } + @NotNull + private HugeLongPriorityQueue createPriorityQueue(long parentSize) { + HugeLongPriorityQueue priorityQueue = minMax == Prim.MAX_OPERATOR + ? HugeLongPriorityQueue.min(parentSize) + : HugeLongPriorityQueue.max(parentSize); + return priorityQueue; + } + @Override public void release() { graph = null; From 038abb05f7cbac288963da3d1840c4708572babb Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 16 Jan 2023 12:36:22 +0100 Subject: [PATCH 039/400] Fix bug with parallel edges --- .../java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 23105bdaac8..55e173d728f 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -78,13 +78,14 @@ public SpanningTree compute() { HugeLongArray parent = spanningTree.parentArray(); long parentSize = parent.size(); HugeLongPriorityQueue priorityQueue = createPriorityQueue(parentSize); + progressTracker.beginSubTask(parentSize); for (long i = 0; i < parentSize && terminationFlag.running(); i++) { long p = parent.get(i); if (p == -1) { continue; } - priorityQueue.add(i, graph.relationshipProperty(p, i, 0.0D)); + priorityQueue.add(i, spanningTree.costToParent(i)); progressTracker.logProgress(); } progressTracker.endSubTask(); From b7dbf6af774c5c196dc4a03ea9d8a6be01bf4fa8 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 16 Jan 2023 17:43:03 +0100 Subject: [PATCH 040/400] Add approach 1: cut leaf --- .../gds/impl/spanningtree/KSpanningTree.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 55e173d728f..014c5191aca 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -116,4 +116,65 @@ public void release() { graph = null; spanningTree = null; } + + private SpanningTree cutLeafApproach(SpanningTree spanningTree) { + //this approach cuts a leaf at each step (remaining graph is always corrected) + //so we can just cut the most expensive leaf at each step + var priorityQueue = createPriorityQueue(graph.nodeCount()); + HugeLongArray degree = HugeLongArray.newArray(graph.nodeCount()); + double startNodeRelationshipCost = -1.0; + long startNodeSingleChild = -1; + HugeLongArray parent = spanningTree.parentArray(); + + for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { + var nodeParent = spanningTree.parent(nodeId); + if (nodeParent != -1) { + degree.set(nodeParent, degree.get(nodeParent) + 1); + if (nodeParent == startNodeId) { //start-node needs special care because it's parent is -1 + startNodeSingleChild = nodeId; + startNodeRelationshipCost = spanningTree.costToParent(nodeId); + } + degree.set(nodeId, degree.get(nodeId) + 1); + } + } + + for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { + if (degree.get(nodeId) == 1) { + double relevantCost = (nodeId == startNodeId) ? + startNodeRelationshipCost : spanningTree.costToParent(nodeId); + priorityQueue.add(nodeId, relevantCost); + } + } + long numberOfDeletions = spanningTree.effectiveNodeCount() - k; + + for (long i = 0; i < numberOfDeletions; ++i) { + var nextNode = priorityQueue.pop(); + long affectedNode = -1; + if (nextNode == startNodeId) { + parent.set(startNodeSingleChild, -1); + affectedNode = startNodeSingleChild; + //should also upd costArray + } else { + affectedNode = parent.get(nextNode); + parent.set(nextNode, -1); + //should also upd costArray + } + degree.set(affectedNode, degree.get(affectedNode) - 1); + if (degree.get(affectedNode) == 1) { + if (affectedNode != startNodeId) { + priorityQueue.add(affectedNode, spanningTree.costToParent(affectedNode)); + } else { + //this can only happen once so the O(n) cost is not a big overhead + for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { + if (parent.get(nodeId) == startNodeId) { + priorityQueue.add(startNodeId, spanningTree.costToParent(nodeId)); + startNodeSingleChild = nodeId; + break; + } + } + } + } + } + return spanningTree; + } } From eeb6b8478bde6770d0a009bbcfc8565aae7b7e4d Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 16 Jan 2023 18:32:45 +0100 Subject: [PATCH 041/400] Add approach 2: modifiable MST --- .../gds/impl/spanningtree/KSpanningTree.java | 99 +++++++++++++++++-- 1 file changed, 93 insertions(+), 6 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 014c5191aca..41e5f1a5e93 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.impl.spanningtree; +import com.carrotsearch.hppc.BitSet; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; @@ -77,7 +78,7 @@ public SpanningTree compute() { SpanningTree spanningTree = prim.compute(); HugeLongArray parent = spanningTree.parentArray(); long parentSize = parent.size(); - HugeLongPriorityQueue priorityQueue = createPriorityQueue(parentSize); + HugeLongPriorityQueue priorityQueue = createPriorityQueue(parentSize, true); progressTracker.beginSubTask(parentSize); for (long i = 0; i < parentSize && terminationFlag.running(); i++) { @@ -104,8 +105,13 @@ public SpanningTree compute() { } @NotNull - private HugeLongPriorityQueue createPriorityQueue(long parentSize) { - HugeLongPriorityQueue priorityQueue = minMax == Prim.MAX_OPERATOR + private HugeLongPriorityQueue createPriorityQueue(long parentSize, boolean reverse) { + //TODO: Don't forget to check this (something is wrong now but will fix tomorrowo) + boolean condition = minMax == Prim.MAX_OPERATOR; + if (reverse) { + condition = !condition; + } + HugeLongPriorityQueue priorityQueue = condition ? HugeLongPriorityQueue.min(parentSize) : HugeLongPriorityQueue.max(parentSize); return priorityQueue; @@ -120,7 +126,7 @@ public void release() { private SpanningTree cutLeafApproach(SpanningTree spanningTree) { //this approach cuts a leaf at each step (remaining graph is always corrected) //so we can just cut the most expensive leaf at each step - var priorityQueue = createPriorityQueue(graph.nodeCount()); + var priorityQueue = createPriorityQueue(graph.nodeCount(), true); HugeLongArray degree = HugeLongArray.newArray(graph.nodeCount()); double startNodeRelationshipCost = -1.0; long startNodeSingleChild = -1; @@ -153,11 +159,11 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { if (nextNode == startNodeId) { parent.set(startNodeSingleChild, -1); affectedNode = startNodeSingleChild; - //should also upd costArray + //TODO: should also upd costArray } else { affectedNode = parent.get(nextNode); parent.set(nextNode, -1); - //should also upd costArray + //TODO: should also upd costArray } degree.set(affectedNode, degree.get(affectedNode) - 1); if (degree.get(affectedNode) == 1) { @@ -177,4 +183,85 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { } return spanningTree; } + + private SpanningTree growApproach(SpanningTree spanningTree) { + + //this approach grows gradually the MST found in the previous step + //when it is about to get larger than K, we crop the current worst leaf if the new value to be added + // is actually any smaller + + + //TODO: Handle to be able to delete startNode as well (not much different from above approach) + + HugeLongArray outDegree = HugeLongArray.newArray(graph.nodeCount()); + + HugeLongArray parent = spanningTree.parentArray(); + + var priorityQueue = createPriorityQueue(graph.nodeCount(), false); + var toTrim = createPriorityQueue(graph.nodeCount(), true); + + BitSet exterior = new BitSet(graph.nodeCount()); + priorityQueue.add(startNodeId, 0); + long nodesInTree = 0; + while (true) { + long node = priorityQueue.pop(); + long nodeParent = spanningTree.parent(node); + + boolean nodeAdded = false; + if (nodesInTree < k) { //if we are smaller, we can just add it no problemo + nodesInTree++; + nodeAdded = true; + } else { + while (!exterior.get(toTrim.top())) { //not valid frontier nodes anymore, just ignore + toTrim.pop(); + } + boolean shouldMove = moveMakesSense(priorityQueue.cost(node), toTrim.cost(toTrim.top()), minMax); + + if (shouldMove && nodeParent != toTrim.top()) { + //we cannot add it, if it's parent is the one who we're going to kill next, right? + nodeAdded = true; + long popped = toTrim.pop(); + long poppedPopps = parent.get(popped); + parent.set(popped, -1); //this guy bites the dust completely... + if (poppedPopps != -1) { + if (outDegree.get(poppedPopps) == 0) { //...and his parent might become open to deletion soon + toTrim.add(poppedPopps, spanningTree.costToParent(poppedPopps)); + exterior.set(poppedPopps); //if so add it to the reverse p.q + } + } + + } + } + if (nodeAdded) { + + outDegree.set(nodeParent, outDegree.get(nodeParent) + 1); + exterior.clear(nodeParent); + + graph.forEachRelationship(node, 1.0, (s, t, w) -> { + if (parent.get(t) == s) { + //TODO: doing this only on the tree for now for simplicity + //cause its 18h and I do not want to think more :D + //might be able to work on all graph edges not just for those in the tree + + if (!priorityQueue.containsElement(t)) { + priorityQueue.add(t, spanningTree.costToParent(t)); + } + + } + return true; + }); + } + } + + } + + private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator minMax) { + if (minMax == Prim.MAX_OPERATOR) { + return cost1 > cost2; + } else { + return cost1 < cost2; + } + } } + + From 77389c2bf5058edc3b5721dd74ca1c9c50e5a2e0 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 16 Jan 2023 18:35:47 +0100 Subject: [PATCH 042/400] Add the propotype of a method that runs two techniques and yields the best --- .../gds/impl/spanningtree/KSpanningTree.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 41e5f1a5e93..ffaa059ae6d 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -262,6 +262,21 @@ private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator m return cost1 < cost2; } } + + private SpanningTree combineApproach(SpanningTree tree) { + + var spanningTree1 = cutLeafApproach(tree); //should clone 'tree' + var spanningTree2 = growApproach(tree); + + //TODO: Update totalWeight in the two methods + if (spanningTree1.totalWeight() > spanningTree2.totalWeight()) { + return (minMax == Prim.MAX_OPERATOR) ? spanningTree1 : spanningTree2; + } else { + return (minMax == Prim.MAX_OPERATOR) ? spanningTree2 : spanningTree1; + + } + + } } From a32fa69a6bb29e781ba650e1d26c9b35380d6d78 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 16 Jan 2023 18:48:02 +0100 Subject: [PATCH 043/400] Leave some comments to discuss tomorrow if the subject arrives at it --- .../gds/impl/spanningtree/KSpanningTree.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index ffaa059ae6d..dabef59644b 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -264,7 +264,27 @@ private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator m } private SpanningTree combineApproach(SpanningTree tree) { - + /* + * these are quick and fast techniques of returning a k-Tree without having to worry about not + * ending up with a tree of k nodes *1, which was the main reason why the existing algorithm did not work. + * + * Teh first approach just cuts leaves from the final MST + * The second approach grows the MST step-by-step and cuts leaves when it needs to trim an edge + * + * Neither of this approach is the best by itself (the approach of cutting arbitrary heavy nodes is also + * plagued by a lot of mistakes) + * but it is not hard to construct situations where one works well and the other does not. + * So I thought of combining the two get the best. + * + * I think we can have this ready for a 2.3.1 patch-release (let's not rush it for thursday; it's alpha after all) + * and for 2.4 we can try do write some more sophisticated methods (to get the optimal answer for a given + * tree you need O(nk^2) time i think so it's maybe not doable*2) + * + * *1: we can modify the LinkCutTree so that we can check size of resulting sub trees after a cut, but for that + * need to relearn the code :) + * *2: and that is not guaranteed to be the optimal answer in general just for that particular tree :) + * + */ var spanningTree1 = cutLeafApproach(tree); //should clone 'tree' var spanningTree2 = growApproach(tree); From f39bc49188d97019238de01004301836e63fffa7 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 16 Jan 2023 21:10:56 +0100 Subject: [PATCH 044/400] Use combined approach and fix code --- .../gds/impl/spanningtree/KSpanningTree.java | 290 +++++++++++------- .../impl/spanningtree/KSpanningTreeTest.java | 104 ++++++- 2 files changed, 287 insertions(+), 107 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index dabef59644b..22a601b31ce 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -20,9 +20,12 @@ package org.neo4j.gds.impl.spanningtree; import com.carrotsearch.hppc.BitSet; +import org.apache.commons.lang3.mutable.MutableDouble; +import org.apache.commons.lang3.mutable.MutableLong; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.utils.paged.HugeDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.queue.HugeLongPriorityQueue; @@ -47,8 +50,6 @@ public class KSpanningTree extends Algorithm { private final long startNodeId; private final long k; - private SpanningTree spanningTree; - public KSpanningTree( Graph graph, DoubleUnaryOperator minMax, @@ -76,42 +77,18 @@ public SpanningTree compute() { prim.setTerminationFlag(getTerminationFlag()); SpanningTree spanningTree = prim.compute(); - HugeLongArray parent = spanningTree.parentArray(); - long parentSize = parent.size(); - HugeLongPriorityQueue priorityQueue = createPriorityQueue(parentSize, true); - - progressTracker.beginSubTask(parentSize); - for (long i = 0; i < parentSize && terminationFlag.running(); i++) { - long p = parent.get(i); - if (p == -1) { - continue; - } - priorityQueue.add(i, spanningTree.costToParent(i)); - progressTracker.logProgress(); - } - progressTracker.endSubTask(); - progressTracker.beginSubTask(k - 1); - // remove until there are k-1 relationships - long numberOfDeletions = spanningTree.effectiveNodeCount() - k; - for (long i = 0; i < numberOfDeletions && terminationFlag.running(); i++) { - long cutNode = priorityQueue.pop(); - parent.set(cutNode, -1); - progressTracker.logProgress(); - } - progressTracker.endSubTask(); - this.spanningTree = prim.getSpanningTree(); - progressTracker.endSubTask(); - return this.spanningTree; + return combineApproach(spanningTree); } @NotNull - private HugeLongPriorityQueue createPriorityQueue(long parentSize, boolean reverse) { - //TODO: Don't forget to check this (something is wrong now but will fix tomorrowo) - boolean condition = minMax == Prim.MAX_OPERATOR; - if (reverse) { - condition = !condition; + private HugeLongPriorityQueue createPriorityQueue(long parentSize, boolean pruning) { + boolean minQueue = minMax == Prim.MIN_OPERATOR; + //if pruning, we remove the worst (max if it's a minimization problem) + //therefore we flip the priority queue + if (pruning) { + minQueue = !minQueue; } - HugeLongPriorityQueue priorityQueue = condition + HugeLongPriorityQueue priorityQueue = minQueue ? HugeLongPriorityQueue.min(parentSize) : HugeLongPriorityQueue.max(parentSize); return priorityQueue; @@ -120,7 +97,15 @@ private HugeLongPriorityQueue createPriorityQueue(long parentSize, boolean rever @Override public void release() { graph = null; - spanningTree = null; + } + + private double init(HugeLongArray parent, HugeDoubleArray costToParent, SpanningTree spanningTree) { + graph.forEachNode((nodeId) -> { + parent.set(nodeId, spanningTree.parent(nodeId)); + costToParent.set(nodeId, spanningTree.costToParent(nodeId)); + return true; + }); + return spanningTree.totalWeight(); } private SpanningTree cutLeafApproach(SpanningTree spanningTree) { @@ -128,84 +113,116 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { //so we can just cut the most expensive leaf at each step var priorityQueue = createPriorityQueue(graph.nodeCount(), true); HugeLongArray degree = HugeLongArray.newArray(graph.nodeCount()); - double startNodeRelationshipCost = -1.0; - long startNodeSingleChild = -1; - HugeLongArray parent = spanningTree.parentArray(); + long root = startNodeId; + double rootCost = -1.0; + long rootChild = -1; + + HugeLongArray parent = HugeLongArray.newArray(graph.nodeCount()); + HugeDoubleArray costToParent = HugeDoubleArray.newArray(graph.nodeCount()); + + double totalCost = init(parent, costToParent, spanningTree); + //calculate degree of each node in MST for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { - var nodeParent = spanningTree.parent(nodeId); + var nodeParent = parent.get(nodeId); if (nodeParent != -1) { degree.set(nodeParent, degree.get(nodeParent) + 1); - if (nodeParent == startNodeId) { //start-node needs special care because it's parent is -1 - startNodeSingleChild = nodeId; - startNodeRelationshipCost = spanningTree.costToParent(nodeId); - } degree.set(nodeId, degree.get(nodeId) + 1); + + if (nodeParent == root) { //root nodes needs special care because parent is -1 + rootChild = nodeId; + rootCost = costToParent.get(nodeId); + } } } - + //add all leafs in priority queue for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { if (degree.get(nodeId) == 1) { - double relevantCost = (nodeId == startNodeId) ? - startNodeRelationshipCost : spanningTree.costToParent(nodeId); + double relevantCost = (nodeId == root) ? + rootCost : + costToParent.get(nodeId); priorityQueue.add(nodeId, relevantCost); } } - long numberOfDeletions = spanningTree.effectiveNodeCount() - k; + long numberOfDeletions = spanningTree.effectiveNodeCount() - k; for (long i = 0; i < numberOfDeletions; ++i) { var nextNode = priorityQueue.pop(); - long affectedNode = -1; - if (nextNode == startNodeId) { - parent.set(startNodeSingleChild, -1); - affectedNode = startNodeSingleChild; - //TODO: should also upd costArray + long affectedNode; + + if (nextNode == root) { + affectedNode = rootChild; + totalCost -= rootCost; + clearNode(rootChild, parent, costToParent); + root = affectedNode; } else { affectedNode = parent.get(nextNode); - parent.set(nextNode, -1); - //TODO: should also upd costArray + totalCost -= costToParent.get(nextNode); + clearNode(nextNode, parent, costToParent); } + degree.set(affectedNode, degree.get(affectedNode) - 1); + double associatedCost = -1; if (degree.get(affectedNode) == 1) { - if (affectedNode != startNodeId) { - priorityQueue.add(affectedNode, spanningTree.costToParent(affectedNode)); - } else { - //this can only happen once so the O(n) cost is not a big overhead - for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { - if (parent.get(nodeId) == startNodeId) { - priorityQueue.add(startNodeId, spanningTree.costToParent(nodeId)); - startNodeSingleChild = nodeId; - break; + if (affectedNode == root) { + //if it is root, we loop at its neighbors to find its single alive child + MutableDouble mutRootCost = new MutableDouble(); + MutableLong mutRootChild = new MutableLong(); + graph.forEachRelationship(root, (s, t) -> { + if (parent.get(t) == s) { + mutRootChild.setValue(t); + mutRootCost.setValue(costToParent.get(t)); + return false; } - } + return true; + }); + rootChild = mutRootChild.longValue(); + rootCost = mutRootCost.doubleValue(); + associatedCost = rootCost; + } else { + //otherwise we just get the info from parent + associatedCost = costToParent.get(affectedNode); } + priorityQueue.add(affectedNode, associatedCost); + } } - return spanningTree; + return new SpanningTree(-1, graph.nodeCount(), k, parent, costToParent, totalCost); } private SpanningTree growApproach(SpanningTree spanningTree) { //this approach grows gradually the MST found in the previous step //when it is about to get larger than K, we crop the current worst leaf if the new value to be added - // is actually any smaller - + // is actually better //TODO: Handle to be able to delete startNode as well (not much different from above approach) HugeLongArray outDegree = HugeLongArray.newArray(graph.nodeCount()); - HugeLongArray parent = spanningTree.parentArray(); + HugeLongArray parent = HugeLongArray.newArray(graph.nodeCount()); + HugeDoubleArray costToParent = HugeDoubleArray.newArray(graph.nodeCount()); + init(parent, costToParent, spanningTree); + double totalCost = 0; var priorityQueue = createPriorityQueue(graph.nodeCount(), false); var toTrim = createPriorityQueue(graph.nodeCount(), true); + //priority-queue does not have a remove method so we need something to know if a node is still a leaf or not BitSet exterior = new BitSet(graph.nodeCount()); + //at any point, the tree has a root we mark its neighbors in this bitset to avoid looping to find them + BitSet rootNodeAdjacent = new BitSet(graph.nodeCount()); + //we just save which nodes are in the final output and not (just to do clean-up; probably be avoided) + BitSet included = new BitSet(graph.nodeCount()); + priorityQueue.add(startNodeId, 0); + long root = startNodeId; //current root is startNodeId long nodesInTree = 0; - while (true) { - long node = priorityQueue.pop(); - long nodeParent = spanningTree.parent(node); + while (!priorityQueue.isEmpty()) { + long node = priorityQueue.top(); + double associatedCost = priorityQueue.cost(node); + priorityQueue.pop(); + long nodeParent = parent.get(node); boolean nodeAdded = false; if (nodesInTree < k) { //if we are smaller, we can just add it no problemo @@ -213,36 +230,90 @@ private SpanningTree growApproach(SpanningTree spanningTree) { nodeAdded = true; } else { while (!exterior.get(toTrim.top())) { //not valid frontier nodes anymore, just ignore - toTrim.pop(); + toTrim.pop(); //as we said, pq does not have a direct remove method + } + var nodeToTrim = toTrim.top(); //a leaf node with worst cost + if (parent.get(node) == nodeToTrim) { + //we cannot add it, if we're supposed to remove its parent + //TODO: should be totally feasible to consider the 2nd worst then. + continue; } - boolean shouldMove = moveMakesSense(priorityQueue.cost(node), toTrim.cost(toTrim.top()), minMax); - if (shouldMove && nodeParent != toTrim.top()) { - //we cannot add it, if it's parent is the one who we're going to kill next, right? + boolean shouldMove = moveMakesSense(associatedCost, toTrim.cost(nodeToTrim), minMax); + + if (shouldMove) { nodeAdded = true; - long popped = toTrim.pop(); - long poppedPopps = parent.get(popped); - parent.set(popped, -1); //this guy bites the dust completely... - if (poppedPopps != -1) { - if (outDegree.get(poppedPopps) == 0) { //...and his parent might become open to deletion soon - toTrim.add(poppedPopps, spanningTree.costToParent(poppedPopps)); - exterior.set(poppedPopps); //if so add it to the reverse p.q + + double value = toTrim.cost(nodeToTrim); + toTrim.pop(); + + long parentOfTrimmed = parent.get(nodeToTrim); + included.clear(nodeToTrim); //nodeToTrim is removed from the answer + clearNode(nodeToTrim, parent, costToParent); + totalCost -= value; //as well as its cost from the solution + + if (parentOfTrimmed != -1) { //we are not removing the actual root + //reduce degree of parent + outDegree.set(parentOfTrimmed, outDegree.get(parentOfTrimmed) - 1); + long affectedNode = -1; + double affectedCost = -1; + long parentDegree = outDegree.get(parentOfTrimmed); + if (parentOfTrimmed == root) { + rootNodeAdjacent.clear(nodeToTrim); + if (parentDegree == 1) { //it is a leaf + assert rootNodeAdjacent.cardinality() == 1; + var rootChild = rootNodeAdjacent.nextSetBit(0); + affectedNode = root; + affectedCost = costToParent.get(rootChild); + } + } else { + if (parentDegree == 0) { + affectedNode = parentOfTrimmed; + affectedCost = costToParent.get(parentOfTrimmed); + } + } + if (affectedNode != -1) { + toTrim.add(affectedNode, affectedCost); + exterior.set(affectedNode); + } + } else { + //the root is removed, long live the new root! + assert rootNodeAdjacent.cardinality() == 1; + var newRoot = rootNodeAdjacent.nextSetBit(0); + rootNodeAdjacent.clear(); //empty everything + graph.forEachRelationship(newRoot, (s, t) -> { + if (parent.get(t) == s) { + rootNodeAdjacent.set(t); + } + return true; + }); + root = newRoot; + clearNode(root, parent, costToParent); + //see if root is a degree-1 to add to exterior + if (outDegree.get(root) == 1) { + var rootChild = rootNodeAdjacent.nextSetBit(0); + priorityQueue.add(rootChild, costToParent.get(rootChild)); + exterior.set(root); } } - } } if (nodeAdded) { - - outDegree.set(nodeParent, outDegree.get(nodeParent) + 1); - exterior.clear(nodeParent); + included.set(node); + totalCost += associatedCost; + if (nodeParent == root) { + rootNodeAdjacent.set(node); + } + if (node != root) { + outDegree.set(nodeParent, outDegree.get(nodeParent) + 1); + exterior.clear(nodeParent); + } + toTrim.add(node, associatedCost); + exterior.set(node); graph.forEachRelationship(node, 1.0, (s, t, w) -> { if (parent.get(t) == s) { - //TODO: doing this only on the tree for now for simplicity - //cause its 18h and I do not want to think more :D - //might be able to work on all graph edges not just for those in the tree - + //TODO: work's only on mst edges for now (should be doable to re-find an k-MST from whole graph) if (!priorityQueue.containsElement(t)) { priorityQueue.add(t, spanningTree.costToParent(t)); } @@ -250,11 +321,28 @@ private SpanningTree growApproach(SpanningTree spanningTree) { } return true; }); + } else { + clearNode(node, parent, costToParent); + } } + //post-processing step: anything not touched is reset to -1 + graph.forEachNode(nodeId -> { + if (!included.get(nodeId)) { + clearNode(nodeId, parent, costToParent); + } + return true; + }); + + return new SpanningTree(-1, graph.nodeCount(), k, parent, costToParent, totalCost); } + private void clearNode(long node, HugeLongArray parent, HugeDoubleArray costToParent) { + parent.set(node, -1); + costToParent.set(node, -1); + } + private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator minMax) { if (minMax == Prim.MAX_OPERATOR) { return cost1 > cost2; @@ -271,24 +359,20 @@ private SpanningTree combineApproach(SpanningTree tree) { * Teh first approach just cuts leaves from the final MST * The second approach grows the MST step-by-step and cuts leaves when it needs to trim an edge * - * Neither of this approach is the best by itself (the approach of cutting arbitrary heavy nodes is also - * plagued by a lot of mistakes) - * but it is not hard to construct situations where one works well and the other does not. - * So I thought of combining the two get the best. + * Neither of this approach is the best by itself + * (the approach of cutting the heaviest also has its shares of issues ofc) * - * I think we can have this ready for a 2.3.1 patch-release (let's not rush it for thursday; it's alpha after all) - * and for 2.4 we can try do write some more sophisticated methods (to get the optimal answer for a given - * tree you need O(nk^2) time i think so it's maybe not doable*2) - * - * *1: we can modify the LinkCutTree so that we can check size of resulting sub trees after a cut, but for that - * need to relearn the code :) - * *2: and that is not guaranteed to be the optimal answer in general just for that particular tree :) + * but it is not hard to construct situations where one works well and the other does not. + * So I thought of combining the two and return the best * + * This is supposed to be a quick fix to eliminate the bug asap. When we work next time on k-MST, + * we can come up with something more sophisticated (there's optimal algorithms for dealing with subtree; + * but they take O(nk^2) so maybe not good). Otherwise, it's np-complete so... */ - var spanningTree1 = cutLeafApproach(tree); //should clone 'tree' + var spanningTree1 = cutLeafApproach(tree); var spanningTree2 = growApproach(tree); + System.out.println(spanningTree1.totalWeight() + " " + spanningTree2.totalWeight()); - //TODO: Update totalWeight in the two methods if (spanningTree1.totalWeight() > spanningTree2.totalWeight()) { return (minMax == Prim.MAX_OPERATOR) ? spanningTree1 : spanningTree2; } else { diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java index 46250e1142c..5af6050319c 100644 --- a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java @@ -22,6 +22,8 @@ import org.apache.commons.lang3.mutable.MutableLong; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -154,8 +156,9 @@ void shouldProduceSingleConnectedTree() { assertThat(nodesInTree.size()).isEqualTo(k); } - @Test - void shouldProduceSingleTreeWithKMinusOneEdges() { + @ParameterizedTest + @CsvSource({"2,1.0", "3,2.0"}) + void shouldProduceSingleTreeWithKMinusOneEdges(int k, double expected) { var factory = GdlFactory.of("CREATE" + " (a:Node)" + ", (b:Node)" + @@ -172,13 +175,12 @@ void shouldProduceSingleTreeWithKMinusOneEdges() { var graph = factory.build().getUnion(); var startNode = factory.nodeId("a"); - var k = 2; var spanningTree = new KSpanningTree( graph, Prim.MIN_OPERATOR, startNode, - 2, + k, ProgressTracker.NULL_TRACKER ).compute(); @@ -189,5 +191,99 @@ void shouldProduceSingleTreeWithKMinusOneEdges() { }); assertThat(counter.getValue()).isEqualTo(k - 1); + + assertThat(spanningTree.totalWeight()).isEqualTo(expected); + } + + @Test + void worstCaseForCuttingHeaviest() { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (f:Node)" + + ", (g:Node)" + + ", (h:Node)" + + ", (a)-[:TYPE {cost: 9.0}]->(b)" + + ", (b)-[:TYPE {cost: 9.0}]->(c)" + + ", (c)-[:TYPE {cost: 0.0}]->(d)" + + ", (d)-[:TYPE {cost: 10.0}]->(e)" + + ", (e)-[:TYPE {cost: 0.0}]->(f)" + + ", (f)-[:TYPE {cost: 9.0}]->(g)" + + ", (g)-[:TYPE {cost: 9.0}]->(h)" + + ); + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("a"); + + + var spanningTree = new KSpanningTree( + graph, + Prim.MIN_OPERATOR, + startNode, + 4, + ProgressTracker.NULL_TRACKER + ).compute(); + + var counter = new MutableLong(0); + spanningTree.forEach((__, ___, ____) -> { + counter.add(1); + return true; + }); + + assertThat(counter.getValue()).isEqualTo(4 - 1); + assertThat(spanningTree.totalWeight()).isEqualTo(10.0); + //here a 'cut-heaviest' approach would begin by cutting edge 10. + //this would prune the tree into two smaller subtrees of 4 nodes each + //adn thus return a 18 despite the optimal being 10. + //this is one of the many situations where cut-heavy-fails + + } + + @Test + void worstCaseForPruningLeaves() { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (f:Node)" + + ", (g:Node)" + + ", (a)-[:TYPE {cost: 9.0}]->(b)" + + ", (b)-[:TYPE {cost: 0.0}]->(c)" + + ", (c)-[:TYPE {cost: 0.0}]->(d)" + + ", (a)-[:TYPE {cost: 1.0}]->(e)" + + ", (e)-[:TYPE {cost: 1.0}]->(f)" + + ", (f)-[:TYPE {cost: 1.0}]->(g)" + + ); + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("a"); + + + var spanningTree = new KSpanningTree( + graph, + Prim.MIN_OPERATOR, + startNode, + 4, + ProgressTracker.NULL_TRACKER + ).compute(); + + var counter = new MutableLong(0); + spanningTree.forEach((__, ___, ____) -> { + counter.add(1); + return true; + }); + + assertThat(counter.getValue()).isEqualTo(4 - 1); + assertThat(spanningTree.totalWeight()).isEqualTo(3.0); + //here a bad case for pruning just leaves + /// edge weight should be eliminated for the final solution but it is not because + //its leaves are not good. + } + } From c3b1d74a09545c1a6db0099ecbf8f6de64d44743 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 17 Jan 2023 12:49:47 +0100 Subject: [PATCH 045/400] Add some semblance of progress tracking --- .../gds/impl/spanningtree/KSpanningTree.java | 15 +++++-- .../KSpanningTreeAlgorithmFactory.java | 7 +-- .../impl/spanningtree/KSpanningTreeTest.java | 43 +++++++++++++++++++ 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 22a601b31ce..cc20822d28c 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -77,7 +77,10 @@ public SpanningTree compute() { prim.setTerminationFlag(getTerminationFlag()); SpanningTree spanningTree = prim.compute(); - return combineApproach(spanningTree); + + var outputTree = combineApproach(spanningTree); + progressTracker.endSubTask(); + return outputTree; } @NotNull @@ -121,7 +124,9 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { HugeDoubleArray costToParent = HugeDoubleArray.newArray(graph.nodeCount()); double totalCost = init(parent, costToParent, spanningTree); + long numberOfDeletions = spanningTree.effectiveNodeCount() - k; + progressTracker.beginSubTask(numberOfDeletions); //calculate degree of each node in MST for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { var nodeParent = parent.get(nodeId); @@ -145,7 +150,6 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { } } - long numberOfDeletions = spanningTree.effectiveNodeCount() - k; for (long i = 0; i < numberOfDeletions; ++i) { var nextNode = priorityQueue.pop(); long affectedNode; @@ -184,9 +188,10 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { associatedCost = costToParent.get(affectedNode); } priorityQueue.add(affectedNode, associatedCost); - } + progressTracker.logProgress(); } + progressTracker.endSubTask(); return new SpanningTree(-1, graph.nodeCount(), k, parent, costToParent, totalCost); } @@ -218,8 +223,10 @@ private SpanningTree growApproach(SpanningTree spanningTree) { priorityQueue.add(startNodeId, 0); long root = startNodeId; //current root is startNodeId long nodesInTree = 0; + progressTracker.beginSubTask(graph.nodeCount()); while (!priorityQueue.isEmpty()) { long node = priorityQueue.top(); + progressTracker.logProgress(); double associatedCost = priorityQueue.cost(node); priorityQueue.pop(); long nodeParent = parent.get(node); @@ -333,7 +340,7 @@ private SpanningTree growApproach(SpanningTree spanningTree) { } return true; }); - + progressTracker.endSubTask(); return new SpanningTree(-1, graph.nodeCount(), k, parent, costToParent, totalCost); } diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java index e0e3b7bba99..0a6e8748c40 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java @@ -52,9 +52,10 @@ public Task progressTask( ) { return Tasks.task( taskName(), - Tasks.leaf("SpanningTree", graph.nodeCount()), - Tasks.leaf("Add relationship weights"), - Tasks.leaf("Remove relationships") + Tasks.leaf("SpanningTree", graph.relationshipCount()), + Tasks.leaf("Remove relationships 1"), + Tasks.leaf("Remove relationships 2") + ); } diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java index 5af6050319c..61fa3d1acaa 100644 --- a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java @@ -25,7 +25,11 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.neo4j.gds.Orientation; +import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -37,6 +41,8 @@ import java.util.HashSet; import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.assertj.Extractors.removingThreadId; +import static org.neo4j.gds.assertj.Extractors.replaceTimings; /** * 1 @@ -286,4 +292,41 @@ void worstCaseForPruningLeaves() { } + @Test + void shouldLogProgress() { + var config = KSpanningTreeBaseConfigImpl.builder().sourceNode(idFunction.of("a")).k(2).build(); + var factory = new KSpanningTreeAlgorithmFactory<>(); + var log = Neo4jProxy.testLog(); + var progressTracker = new TestProgressTracker( + factory.progressTask(graph, config), + log, + 1, + EmptyTaskRegistryFactory.INSTANCE + ); + factory.build(graph, config, progressTracker).compute(); + assertThat(log.getMessages(TestLog.INFO)) + .extracting(removingThreadId()) + .extracting(replaceTimings()) + .containsExactly( + "KSpanningTree :: Start", + "KSpanningTree :: SpanningTree :: Start", + "KSpanningTree :: SpanningTree 30%", + "KSpanningTree :: SpanningTree 50%", + "KSpanningTree :: SpanningTree 80%", + "KSpanningTree :: SpanningTree 100%", + "KSpanningTree :: SpanningTree :: Finished", + "KSpanningTree :: Remove relationships 1 :: Start", + "KSpanningTree :: Remove relationships 1 50%", + "KSpanningTree :: Remove relationships 1 100%", + "KSpanningTree :: Remove relationships 1 :: Finished", + "KSpanningTree :: Remove relationships 2 :: Start", + "KSpanningTree :: Remove relationships 2 20%", + "KSpanningTree :: Remove relationships 2 40%", + "KSpanningTree :: Remove relationships 2 60%", + "KSpanningTree :: Remove relationships 2 100%", + "KSpanningTree :: Remove relationships 2 :: Finished", + "KSpanningTree :: Finished" + ); + } + } From 586b3f4d30c849d1379b8f232162d2e6d5a8a0e0 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 17 Jan 2023 13:56:46 +0100 Subject: [PATCH 046/400] No text --- .../gds/impl/spanningtree/KSpanningTree.java | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index cc20822d28c..99a9ee38be6 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -359,26 +359,9 @@ private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator m } private SpanningTree combineApproach(SpanningTree tree) { - /* - * these are quick and fast techniques of returning a k-Tree without having to worry about not - * ending up with a tree of k nodes *1, which was the main reason why the existing algorithm did not work. - * - * Teh first approach just cuts leaves from the final MST - * The second approach grows the MST step-by-step and cuts leaves when it needs to trim an edge - * - * Neither of this approach is the best by itself - * (the approach of cutting the heaviest also has its shares of issues ofc) - * - * but it is not hard to construct situations where one works well and the other does not. - * So I thought of combining the two and return the best - * - * This is supposed to be a quick fix to eliminate the bug asap. When we work next time on k-MST, - * we can come up with something more sophisticated (there's optimal algorithms for dealing with subtree; - * but they take O(nk^2) so maybe not good). Otherwise, it's np-complete so... - */ + var spanningTree1 = cutLeafApproach(tree); var spanningTree2 = growApproach(tree); - System.out.println(spanningTree1.totalWeight() + " " + spanningTree2.totalWeight()); if (spanningTree1.totalWeight() > spanningTree2.totalWeight()) { return (minMax == Prim.MAX_OPERATOR) ? spanningTree1 : spanningTree2; From 597be2957a8fc8ade11a64298f1b161505bb0872 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 17 Jan 2023 14:43:30 +0100 Subject: [PATCH 047/400] Addressing Veselin's comments and some bug fixing Co-authored-by: Veselin Nikolov --- .../neo4j/gds/spanningtree/SpanningTree.java | 4 +- .../gds/impl/spanningtree/KSpanningTree.java | 68 +++++++++++-------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java index 091815c1bf0..3a1716cc8aa 100644 --- a/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java +++ b/algo/src/main/java/org/neo4j/gds/spanningtree/SpanningTree.java @@ -74,8 +74,8 @@ public double costToParent(long nodeId) { public void forEach(RelationshipWithPropertyConsumer consumer) { for (int i = 0; i < nodeCount; i++) { - final long parent = this.parent.get(i); - final double cost = this.costToParent(i); + long parent = this.parent.get(i); + double cost = this.costToParent(i); if (parent == -1) { continue; } diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 99a9ee38be6..bddc459ea00 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -99,6 +99,7 @@ private HugeLongPriorityQueue createPriorityQueue(long parentSize, boolean pruni @Override public void release() { + graph.release(); graph = null; } @@ -153,13 +154,14 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { for (long i = 0; i < numberOfDeletions; ++i) { var nextNode = priorityQueue.pop(); long affectedNode; - - if (nextNode == root) { + + if (nextNode == root) { //the affecte node is its single child affectedNode = rootChild; totalCost -= rootCost; clearNode(rootChild, parent, costToParent); + clearNode(root, parent, costToParent); root = affectedNode; - } else { + } else { //the affected node is its paret affectedNode = parent.get(nextNode); totalCost -= costToParent.get(nextNode); clearNode(nextNode, parent, costToParent); @@ -167,7 +169,7 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { degree.set(affectedNode, degree.get(affectedNode) - 1); double associatedCost = -1; - if (degree.get(affectedNode) == 1) { + if (degree.get(affectedNode) == 1) { //it becomes a leaf if (affectedNode == root) { //if it is root, we loop at its neighbors to find its single alive child MutableDouble mutRootCost = new MutableDouble(); @@ -192,7 +194,7 @@ private SpanningTree cutLeafApproach(SpanningTree spanningTree) { progressTracker.logProgress(); } progressTracker.endSubTask(); - return new SpanningTree(-1, graph.nodeCount(), k, parent, costToParent, totalCost); + return new SpanningTree(root, graph.nodeCount(), k, parent, costToParent, totalCost); } private SpanningTree growApproach(SpanningTree spanningTree) { @@ -201,8 +203,6 @@ private SpanningTree growApproach(SpanningTree spanningTree) { //when it is about to get larger than K, we crop the current worst leaf if the new value to be added // is actually better - //TODO: Handle to be able to delete startNode as well (not much different from above approach) - HugeLongArray outDegree = HugeLongArray.newArray(graph.nodeCount()); HugeLongArray parent = HugeLongArray.newArray(graph.nodeCount()); @@ -213,11 +213,12 @@ private SpanningTree growApproach(SpanningTree spanningTree) { var priorityQueue = createPriorityQueue(graph.nodeCount(), false); var toTrim = createPriorityQueue(graph.nodeCount(), true); - //priority-queue does not have a remove method so we need something to know if a node is still a leaf or not + //priority-queue does not have a remove method + // so we need something to know if a node is still a leaf or not BitSet exterior = new BitSet(graph.nodeCount()); //at any point, the tree has a root we mark its neighbors in this bitset to avoid looping to find them BitSet rootNodeAdjacent = new BitSet(graph.nodeCount()); - //we just save which nodes are in the final output and not (just to do clean-up; probably be avoided) + //we just save which nodes are in the final output and not (just to do clean-up; probably can be avoided) BitSet included = new BitSet(graph.nodeCount()); priorityQueue.add(startNodeId, 0); @@ -259,66 +260,75 @@ private SpanningTree growApproach(SpanningTree spanningTree) { clearNode(nodeToTrim, parent, costToParent); totalCost -= value; //as well as its cost from the solution - if (parentOfTrimmed != -1) { //we are not removing the actual root + if (root != nodeToTrim) { //we are not removing the actual root //reduce degree of parent outDegree.set(parentOfTrimmed, outDegree.get(parentOfTrimmed) - 1); long affectedNode = -1; double affectedCost = -1; - long parentDegree = outDegree.get(parentOfTrimmed); - if (parentOfTrimmed == root) { - rootNodeAdjacent.clear(nodeToTrim); - if (parentDegree == 1) { //it is a leaf + long parentOutDegree = outDegree.get(parentOfTrimmed); + if (parentOfTrimmed == root) { //if its parent is the root + rootNodeAdjacent.clear(nodeToTrim); //remove the trimmed child + if (parentOutDegree == 1) { //root becomes a leaf assert rootNodeAdjacent.cardinality() == 1; + //get the single sole child of root var rootChild = rootNodeAdjacent.nextSetBit(0); affectedNode = root; affectedCost = costToParent.get(rootChild); } } else { - if (parentDegree == 0) { + if (parentOutDegree == 0) { //if parent is a leaf affectedNode = parentOfTrimmed; affectedCost = costToParent.get(parentOfTrimmed); } } - if (affectedNode != -1) { - toTrim.add(affectedNode, affectedCost); - exterior.set(affectedNode); + if (affectedNode != -1) { //if a node has been converted to a leaf + toTrim.add(affectedNode, affectedCost); //add it to pq + exterior.set(affectedNode); //and mark it in the exterior } } else { //the root is removed, long live the new root! assert rootNodeAdjacent.cardinality() == 1; + //the new root is the single sole child of old root var newRoot = rootNodeAdjacent.nextSetBit(0); rootNodeAdjacent.clear(); //empty everything + //find the children of the new root (this can happen once per node) graph.forEachRelationship(newRoot, (s, t) -> { - if (parent.get(t) == s) { + //relevant are only those nodes which are currently + //in the k-tree + if (parent.get(t) == s && included.get(t)) { rootNodeAdjacent.set(t); } return true; }); root = newRoot; + //set it as root clearNode(root, parent, costToParent); - //see if root is a degree-1 to add to exterior + //check if root is a degree-1 to add to exterior if (outDegree.get(root) == 1) { + //get single child var rootChild = rootNodeAdjacent.nextSetBit(0); - priorityQueue.add(rootChild, costToParent.get(rootChild)); + priorityQueue.add(root, costToParent.get(rootChild)); exterior.set(root); } } } } if (nodeAdded) { - included.set(node); - totalCost += associatedCost; - if (nodeParent == root) { + included.set(node); // include it in the solution (for now!) + totalCost += associatedCost; //add its associated cost to the weight of tree + if (nodeParent == root) { //if it's parent is the root, update the bitset rootNodeAdjacent.set(node); } - if (node != root) { + if (node != root) { //this only happens for startNode to be fair + //the node's parent gets an update in degree outDegree.set(nodeParent, outDegree.get(nodeParent) + 1); - exterior.clear(nodeParent); + exterior.clear(nodeParent); //and remoed from exterior if included } + //then the node (being a leaf) is added to the trimming priority queu toTrim.add(node, associatedCost); - exterior.set(node); + exterior.set(node); //and the exterior - graph.forEachRelationship(node, 1.0, (s, t, w) -> { + graph.forEachRelationship(node, (s, t) -> { if (parent.get(t) == s) { //TODO: work's only on mst edges for now (should be doable to re-find an k-MST from whole graph) if (!priorityQueue.containsElement(t)) { @@ -341,7 +351,7 @@ private SpanningTree growApproach(SpanningTree spanningTree) { return true; }); progressTracker.endSubTask(); - return new SpanningTree(-1, graph.nodeCount(), k, parent, costToParent, totalCost); + return new SpanningTree(root, graph.nodeCount(), k, parent, costToParent, totalCost); } From 7f618dcb43925f960e6e2d65ba2bcd215d46f521 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 20 Jan 2023 10:09:28 +0100 Subject: [PATCH 048/400] Fix K-spanning tree docTests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- .../KSpanningTreeWriteProcTest.java | 14 ++-- .../neo4j/gds/doc/KSpanningTreeDocTest.java | 48 +++++++++++++ .../k-minimum-weight-spanning-tree.adoc | 69 +++++++++++-------- 3 files changed, 99 insertions(+), 32 deletions(-) create mode 100644 doc-test/src/test/java/org/neo4j/gds/doc/KSpanningTreeDocTest.java diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java index 89095f9143a..e2ad893d7ee 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/spanningtree/KSpanningTreeWriteProcTest.java @@ -30,6 +30,7 @@ import org.neo4j.gds.extension.Neo4jGraph; import java.util.HashMap; +import java.util.HashSet; import static org.assertj.core.api.Assertions.assertThat; @@ -83,16 +84,20 @@ void testMax() { assertThat(row.getNumber("computeMillis").longValue() >= 0).isTrue(); }); - final HashMap communities = new HashMap<>(); + final HashMap communities = new HashMap<>(); + final HashSet distinctCommunities = new HashSet<>(); runQueryWithRowConsumer("MATCH (n) WHERE n.partition IS NOT NULL RETURN n.name as name, n.partition as p", row -> { final String name = row.getString("name"); - final long p = row.getNumber("p").longValue(); + final int p = row.getNumber("p").intValue(); communities.put(name, p); + distinctCommunities.add(p); + }); assertThat(communities).matches(c -> c.get("a").equals(c.get("b")) ^ c.get("c").equals(c.get("d"))); assertThat(communities.get("a")).isNotEqualTo(communities.get("c")); + assertThat(distinctCommunities.size()).isEqualTo(3); } @Test @@ -113,14 +118,15 @@ void testMin() { }); final HashMap communities = new HashMap<>(); - + final HashSet distinctCommunities = new HashSet<>(); runQueryWithRowConsumer("MATCH (n) WHERE n.partition IS NOT NULL RETURN n.name as name, n.partition as p", row -> { final String name = row.getString("name"); final int p = row.getNumber("p").intValue(); communities.put(name, p); + distinctCommunities.add(p); }); assertThat(communities).matches(c -> c.get("a").equals(c.get("d")) ^ c.get("b").equals(c.get("c"))); - assertThat(communities.get("a")).isNotEqualTo(communities.get("b")); + assertThat(distinctCommunities.size()).isEqualTo(3); } } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/KSpanningTreeDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/KSpanningTreeDocTest.java new file mode 100644 index 00000000000..4d11e82ae66 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/KSpanningTreeDocTest.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc; + +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.functions.AsNodeFunc; +import org.neo4j.gds.spanningtree.KSpanningWriteTreeProc; + +import java.util.List; + +class KSpanningTreeDocTest extends SingleFileDocTestBase { + + @Override + protected List> functions() { + return List.of(AsNodeFunc.class); + } + + @Override + protected List> procedures() { + return List.of( + KSpanningWriteTreeProc.class, + GraphProjectProc.class + ); + } + + @Override + protected String adocFile() { + return "pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc"; + } + +} diff --git a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc index 7269360fd17..08bd34ada88 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc @@ -62,7 +62,7 @@ YIELD effectiveNodeCount: Integer, image::mst.png[] .The following will create the sample graph depicted in the figure: -[source, cypher, role=noplay] +[source, cypher, role=noplay setup-query] ---- CREATE (a:Place {id: 'A'}), (b:Place {id: 'B'}), @@ -81,7 +81,7 @@ CREATE (a:Place {id: 'A'}), ---- .The following will project and store a named graph: -[source, cypher, role=noplay] +[source, cypher, role=noplay graph-project-query] ---- CALL gds.graph.project( 'graph', @@ -98,11 +98,14 @@ CALL gds.graph.project( [[algorithms-minimum-weight-spanning-tree-k]] == K-Spanning tree examples +=== Minimum K-Spanning Tree example + In our sample graph we have 5 nodes. -When we ran MST above, we got a 5-minimum spanning tree returned, that covered all five nodes. By setting the `k=3`, we define that we want to get returned a 3-minimum spanning tree that covers 3 nodes and has 2 relationships. .The following will run the k-minimum spanning tree algorithm and write back results: +[role=query-example, no-result=true, group=write-example] +-- [source, cypher, role=noplay] ---- MATCH (n:Place{id: 'D'}) @@ -110,34 +113,42 @@ CALL gds.alpha.kSpanningTree.write('graph', { k: 3, sourceNode: id(n), relationshipWeightProperty: 'cost', - writeProperty:'kminst' + writeProperty:'kmin' }) YIELD preProcessingMillis, computeMillis, writeMillis, effectiveNodeCount RETURN preProcessingMillis,computeMillis,writeMillis, effectiveNodeCount; ---- +-- -.Find nodes that belong to our k-spanning tree result: +[role=query-example, group=write-example] +-- +.The following will find the nodes that belong to our k-spanning tree result: [source, cypher, role=noplay] ---- -MATCH (n:Place) -WITH n.id AS Place, n.kminst AS Partition, count(*) AS count -WHERE count = 3 -RETURN Place, Partition +MATCH (n) +WITH n.kmin AS p, count(n) AS c +WHERE c = 3 +MATCH (n) +WHERE n.kmin = p +RETURN n.id As Place, p as Partition + ---- .Results [opts="header",cols="1,1"] |=== | Place | Partition -| A | 1 -| B | 1 -| C | 1 -| D | 3 -| E | 4 +| "A" | 1 +| "B" | 1 +| "C" | 1 |=== - +-- Nodes A, B, and C are the result 3-minimum spanning tree of our graph. +=== Maximum K-Spanning Tree example + +[role=query-example,no-result=true, group=max-example] +-- .The following will run the k-maximum spanning tree algorithm and write back results: [source, cypher, role=noplay] ---- @@ -146,31 +157,33 @@ CALL gds.alpha.kSpanningTree.write('graph', { k: 3, sourceNode: id(n), relationshipWeightProperty: 'cost', - writeProperty:'kmaxst', - objective: 'maximum', + writeProperty:'kmax', + objective: 'maximum' }) YIELD preProcessingMillis, computeMillis, writeMillis, effectiveNodeCount RETURN preProcessingMillis,computeMillis,writeMillis, effectiveNodeCount; ---- +-- +[role=query-example, group=max-example] +-- .Find nodes that belong to our k-spanning tree result: [source, cypher, role=noplay] ---- -MATCH (n:Place) -WITH n.id AS Place, n.kmaxst AS Partition, count(*) AS count -WHERE count = 3 -RETURN Place, Partition +MATCH (n) +WITH n.kmax AS p, count(n) AS c +WHERE c = 3 +MATCH (n) +WHERE n.kmax = p +RETURN n.id As Place, p as Partition ---- - .Results [opts="header",cols="1,1"] |=== | Place | Partition -| A | 0 -| B | 1 -| C | 3 -| D | 3 -| E | 3 +| "C" | 3 +| "D" | 3 +| "E" | 3 |=== - +-- Nodes C, D, and E are the result 3-maximum spanning tree of our graph. From ff99dd4421ac89e16cff93e3260fa1035f518e33 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 20 Jan 2023 10:33:48 +0100 Subject: [PATCH 049/400] Add syntaxt testing --- .../doc/syntax/KSpanningTreeSyntaxTest.java | 38 +++++++++++++++++++ .../k-minimum-weight-spanning-tree.adoc | 25 +++++++++--- .../specific-configuration.adoc | 4 ++ 3 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 doc-test/src/test/java/org/neo4j/gds/doc/syntax/KSpanningTreeSyntaxTest.java create mode 100644 doc/modules/ROOT/partials/algorithms/k-spanning-tree/specific-configuration.adoc diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/KSpanningTreeSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/KSpanningTreeSyntaxTest.java new file mode 100644 index 00000000000..af085dd7540 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/KSpanningTreeSyntaxTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc.syntax; + +import java.util.List; + +import static org.neo4j.gds.doc.syntax.SyntaxMode.WRITE; + +class KSpanningTreeSyntaxTest extends SyntaxTestBase { + + protected Iterable syntaxModes() { + return List.of( + SyntaxModeMeta.of(WRITE) + ); + } + + @Override + protected String adocFile() { + return "pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc"; + } +} diff --git a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc index 08bd34ada88..d27a1f5cc69 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc @@ -21,6 +21,14 @@ The minimum weight k-Spanning Tree is NP-Hard. The algorithm in the Neo4j GDS Li [[algorithms-minimum-k-weight-spanning-tree-syntax]] == Syntax +include::partial$/algorithms/shared/syntax-intro-named-graph.adoc[] + +.K Spanning Tree syntax per mode +[.tabbed-example, caption = ] +==== + +[.include-with-write] +====== .The following will run the k-spanning tree algorithms and write back results: [source, cypher, role=noplay] ---- @@ -31,19 +39,19 @@ CALL gds.alpha.kSpanningTree.write( YIELD effectiveNodeCount: Integer, preProcessingMillis: Integer, computeMillis: Integer, + postProcessingMillis: Integer, writeMillis: Integer, configuration: Map ---- +include::partial$/algorithms/common-configuration/common-parameters.adoc[] .Configuration [opts="header",cols="1,1,1,1,4"] |=== -| Name | Type | Default | Optional | Description -| k | Integer | null | no | The result is a tree with `k` nodes and `k − 1` relationships. -| sourceNode | Integer | null | no | The start node ID. -| xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. -| writeProperty | String | n/a | no | The partition that a node belongs to. -| objective | String | 'minimum' | yes | If specified, the parameter dictates whether to try and find the minimum or the maximum k-spanning tree. By default, the algorithm looks for the minimum one. Permitted values are 'minimum' and 'maximum'. +| Name | Type | Default | Optional | Description +include::partial$/algorithms/common-configuration/common-write-configuration-entries.adoc[] +include::partial$/algorithms/k-spanning-tree/specific-configuration.adoc[] + |=== .Results @@ -53,8 +61,13 @@ YIELD effectiveNodeCount: Integer, | effectiveNodeCount | Integer | The number of visited nodes. | preProcessingMillis | Integer | Milliseconds for preprocessing the data. | computeMillis | Integer | Milliseconds for running the algorithm. +| postProcessingMillis | Integer | Milliseconds for postprocessing results of the algorithm. | writeMillis | Integer | Milliseconds for writing result data back. +| configuration | Map | The configuration used for running the algorithm. + |=== +====== +==== [[algorithms-minimum-weight-spanning-tree-sample]] == Minimum Weight k-Spanning Tree algorithm examples diff --git a/doc/modules/ROOT/partials/algorithms/k-spanning-tree/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/k-spanning-tree/specific-configuration.adoc new file mode 100644 index 00000000000..4e5ad333b89 --- /dev/null +++ b/doc/modules/ROOT/partials/algorithms/k-spanning-tree/specific-configuration.adoc @@ -0,0 +1,4 @@ +| k | Number | n/a | no | The size of the tree to be returned +| sourceNode | Integer | null | n/a | The starting source node ID. +| xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. +| objective | String | 'minimum' | yes | If specified, the parameter dictates whether to seek a minimum or the maximum weight k-spanning tree. By default, the procedure looks for a minimum weight k-spanning tree. Permitted values are 'minimum' and 'maximum'. From c6d998095318032f6e0a4f98d087995a946a3521 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 20 Jan 2023 10:53:31 +0100 Subject: [PATCH 050/400] Proofreading --- .../k-minimum-weight-spanning-tree.adoc | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc index d27a1f5cc69..815d331ebd3 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc @@ -3,7 +3,7 @@ = Minimum Weight k-Spanning Tree :description: This section describes the Minimum Weight k-Spanning Tree algorithm in the Neo4j Graph Data Science library. :entity: node -:result: spanning tree edge +:result: spanning tree :algorithm: k-Spanning Tree heuristic include::partial$/operations-reference/alpha-note.adoc[] @@ -11,13 +11,18 @@ include::partial$/operations-reference/alpha-note.adoc[] == Introduction -Sometimes, we want to limit the size of our spanning tree result, as we are only interested in finding a smaller tree within the graph, and not one that necessarily spans across all nodes. +Sometimes, we might require a spanning tree(a tree where its nodes are connected with each via a single path) that does not necessarily span all nodes in the graph. The K-Spanning tree heuristic algorithm returns a tree with `k` nodes and `k − 1` relationships. -Our heuristic processes the result found by the Prim algorithm for the Minimum Weight Spanning Tree problem. +Our heuristic processes the result found by Prim's algorithm for the xref:algorithms/minimum-weight-spanning-tree.adoc[Minimum Weight Spanning Tree] problem. +Like Prim, it starts from a given source node, finds a spanning tree for all nodes and then removes nodes using heuristics to produce a tree with 'k' nodes. +Note that the source node will not be necessarily included in the final output as the heuristic tries to find a globally good tree. [[algorithms-k-spanning]] == Considerations -The minimum weight k-Spanning Tree is NP-Hard. The algorithm in the Neo4j GDS Library is therefore not guaranteed to find the optimal answer, but should return good approximation in practice. +The Minimum weight k-Spanning Tree is NP-Hard. The algorithm in the Neo4j GDS Library is therefore not guaranteed to find the optimal answer, but should hopefully return a good approximation in practice. + +Like Prim algorithm, the algorithm focuses only on the component of the source node. If that component has fewer than `k` nodes, it will not look into other components, but will instead return the component. + [[algorithms-minimum-k-weight-spanning-tree-syntax]] == Syntax @@ -46,7 +51,7 @@ YIELD effectiveNodeCount: Integer, include::partial$/algorithms/common-configuration/common-parameters.adoc[] .Configuration -[opts="header",cols="1,1,1,1,4"] +[opts="header",cols="3,2,3m,2,8"] |=== | Name | Type | Default | Optional | Description include::partial$/algorithms/common-configuration/common-write-configuration-entries.adoc[] @@ -72,7 +77,10 @@ include::partial$/algorithms/k-spanning-tree/specific-configuration.adoc[] [[algorithms-minimum-weight-spanning-tree-sample]] == Minimum Weight k-Spanning Tree algorithm examples -image::mst.png[] +:algorithm-name: {algorithm} +:graph-description: road network +:image-file: spanning-tree-graph.svg +include::partial$/algorithms/shared/examples-intro.adoc[] .The following will create the sample graph depicted in the figure: [source, cypher, role=noplay setup-query] @@ -113,8 +121,8 @@ CALL gds.graph.project( === Minimum K-Spanning Tree example -In our sample graph we have 5 nodes. -By setting the `k=3`, we define that we want to get returned a 3-minimum spanning tree that covers 3 nodes and has 2 relationships. +In our sample graph we have 7 nodes. +By setting the `k=3`, we define that we want to find a 3-minimum spanning tree that covers 3 nodes and has 2 relationships. .The following will run the k-minimum spanning tree algorithm and write back results: [role=query-example, no-result=true, group=write-example] @@ -199,4 +207,4 @@ RETURN n.id As Place, p as Partition | "E" | 3 |=== -- -Nodes C, D, and E are the result 3-maximum spanning tree of our graph. +Nodes C, D, and E form a 3-maximum spanning tree of our graph. From 746cae755694957284a9c1e2c3d0529c511f9c89 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 20 Jan 2023 10:58:34 +0100 Subject: [PATCH 051/400] Handle smaller than k components --- .../gds/impl/spanningtree/KSpanningTree.java | 4 ++- .../impl/spanningtree/KSpanningTreeTest.java | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index bddc459ea00..61bdc24fb70 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -369,7 +369,9 @@ private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator m } private SpanningTree combineApproach(SpanningTree tree) { - + if (tree.effectiveNodeCount() < k) { + return tree; + } var spanningTree1 = cutLeafApproach(tree); var spanningTree2 = growApproach(tree); diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java index 61fa3d1acaa..bfe584ceedf 100644 --- a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java @@ -292,6 +292,35 @@ void worstCaseForPruningLeaves() { } + @Test + void shouldWorkForComponentSmallerThanK() { + var factory = GdlFactory.of("CREATE" + + " (a:Node)" + + ", (b:Node)" + + ", (c:Node)" + + ", (d:Node)" + + ", (e:Node)" + + ", (f:Node)" + + ", (g:Node)" + + ", (a)-[:TYPE {cost: 1.0}]->(b)" + + ", (b)-[:TYPE {cost: 1.0}]->(c)" + + ", (c)-[:TYPE {cost: 1.0}]->(d)"); + + var graph = factory.build().getUnion(); + var startNode = factory.nodeId("a"); + + var spanningTree = new KSpanningTree( + graph, + Prim.MIN_OPERATOR, + startNode, + 5, + ProgressTracker.NULL_TRACKER + ).compute(); + + assertThat(spanningTree.effectiveNodeCount()).isEqualTo(4); + + } + @Test void shouldLogProgress() { var config = KSpanningTreeBaseConfigImpl.builder().sourceNode(idFunction.of("a")).k(2).build(); From 2c58f34fc03af459be9bb67ae48ef5a863ff052f Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 20 Jan 2023 11:02:33 +0100 Subject: [PATCH 052/400] minor comment --- .../java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 61bdc24fb70..71e371a478e 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -99,7 +99,6 @@ private HugeLongPriorityQueue createPriorityQueue(long parentSize, boolean pruni @Override public void release() { - graph.release(); graph = null; } @@ -276,7 +275,7 @@ private SpanningTree growApproach(SpanningTree spanningTree) { affectedCost = costToParent.get(rootChild); } } else { - if (parentOutDegree == 0) { //if parent is a leaf + if (parentOutDegree == 0) { //if parent becomes a leaf affectedNode = parentOfTrimmed; affectedCost = costToParent.get(parentOfTrimmed); } From 6de60cabcf5145f3cce963435017203c86d4602d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Mon, 23 Jan 2023 10:43:24 +0100 Subject: [PATCH 053/400] Bump Aura build number --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 37a697a5ccd..b8824f463af 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.0' - gdsAuraVersion = '8' + gdsAuraVersion = '9' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 633455024b5ae408fe6bf98346849e13be7922fa Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 23 Jan 2023 11:15:31 +0100 Subject: [PATCH 054/400] use HLPQ set instead of add for existing heap elements --- .../org/neo4j/gds/impl/spanningtree/KSpanningTree.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 71e371a478e..07902a743e0 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -281,7 +281,12 @@ private SpanningTree growApproach(SpanningTree spanningTree) { } } if (affectedNode != -1) { //if a node has been converted to a leaf - toTrim.add(affectedNode, affectedCost); //add it to pq + if (!toTrim.containsElement(affectedNode)) { + toTrim.add(affectedNode, affectedCost); //add it to pq + } else { + //it is still in the queue, but it is not a leaf anymore, so it's value is obsolete + toTrim.set(affectedNode, affectedCost); + } exterior.set(affectedNode); //and mark it in the exterior } } else { @@ -373,7 +378,7 @@ private SpanningTree combineApproach(SpanningTree tree) { } var spanningTree1 = cutLeafApproach(tree); var spanningTree2 = growApproach(tree); - + System.out.println("Prune Leaf: " + spanningTree1.totalWeight() + " Grow Mst:" + spanningTree2.totalWeight()); if (spanningTree1.totalWeight() > spanningTree2.totalWeight()) { return (minMax == Prim.MAX_OPERATOR) ? spanningTree1 : spanningTree2; } else { From 8bcb76d572330adb477d9fa0f9b3e7adef20dac7 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 23 Jan 2023 11:41:55 +0100 Subject: [PATCH 055/400] Drop first heuristic as being theoretically useful --- .../gds/impl/spanningtree/KSpanningTree.java | 107 +----------------- .../KSpanningTreeAlgorithmFactory.java | 4 +- .../impl/spanningtree/KSpanningTreeTest.java | 78 +++---------- 3 files changed, 19 insertions(+), 170 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 07902a743e0..19b422f8ce8 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -20,8 +20,6 @@ package org.neo4j.gds.impl.spanningtree; import com.carrotsearch.hppc.BitSet; -import org.apache.commons.lang3.mutable.MutableDouble; -import org.apache.commons.lang3.mutable.MutableLong; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; @@ -78,7 +76,7 @@ public SpanningTree compute() { prim.setTerminationFlag(getTerminationFlag()); SpanningTree spanningTree = prim.compute(); - var outputTree = combineApproach(spanningTree); + var outputTree = growApproach(spanningTree); progressTracker.endSubTask(); return outputTree; } @@ -111,97 +109,15 @@ private double init(HugeLongArray parent, HugeDoubleArray costToParent, Spanning return spanningTree.totalWeight(); } - private SpanningTree cutLeafApproach(SpanningTree spanningTree) { - //this approach cuts a leaf at each step (remaining graph is always corrected) - //so we can just cut the most expensive leaf at each step - var priorityQueue = createPriorityQueue(graph.nodeCount(), true); - HugeLongArray degree = HugeLongArray.newArray(graph.nodeCount()); - long root = startNodeId; - double rootCost = -1.0; - long rootChild = -1; - - HugeLongArray parent = HugeLongArray.newArray(graph.nodeCount()); - HugeDoubleArray costToParent = HugeDoubleArray.newArray(graph.nodeCount()); - - double totalCost = init(parent, costToParent, spanningTree); - long numberOfDeletions = spanningTree.effectiveNodeCount() - k; - - progressTracker.beginSubTask(numberOfDeletions); - //calculate degree of each node in MST - for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { - var nodeParent = parent.get(nodeId); - if (nodeParent != -1) { - degree.set(nodeParent, degree.get(nodeParent) + 1); - degree.set(nodeId, degree.get(nodeId) + 1); - - if (nodeParent == root) { //root nodes needs special care because parent is -1 - rootChild = nodeId; - rootCost = costToParent.get(nodeId); - } - } - } - //add all leafs in priority queue - for (long nodeId = 0; nodeId < graph.nodeCount(); ++nodeId) { - if (degree.get(nodeId) == 1) { - double relevantCost = (nodeId == root) ? - rootCost : - costToParent.get(nodeId); - priorityQueue.add(nodeId, relevantCost); - } - } - - for (long i = 0; i < numberOfDeletions; ++i) { - var nextNode = priorityQueue.pop(); - long affectedNode; - - if (nextNode == root) { //the affecte node is its single child - affectedNode = rootChild; - totalCost -= rootCost; - clearNode(rootChild, parent, costToParent); - clearNode(root, parent, costToParent); - root = affectedNode; - } else { //the affected node is its paret - affectedNode = parent.get(nextNode); - totalCost -= costToParent.get(nextNode); - clearNode(nextNode, parent, costToParent); - } - - degree.set(affectedNode, degree.get(affectedNode) - 1); - double associatedCost = -1; - if (degree.get(affectedNode) == 1) { //it becomes a leaf - if (affectedNode == root) { - //if it is root, we loop at its neighbors to find its single alive child - MutableDouble mutRootCost = new MutableDouble(); - MutableLong mutRootChild = new MutableLong(); - graph.forEachRelationship(root, (s, t) -> { - if (parent.get(t) == s) { - mutRootChild.setValue(t); - mutRootCost.setValue(costToParent.get(t)); - return false; - } - return true; - }); - rootChild = mutRootChild.longValue(); - rootCost = mutRootCost.doubleValue(); - associatedCost = rootCost; - } else { - //otherwise we just get the info from parent - associatedCost = costToParent.get(affectedNode); - } - priorityQueue.add(affectedNode, associatedCost); - } - progressTracker.logProgress(); - } - progressTracker.endSubTask(); - return new SpanningTree(root, graph.nodeCount(), k, parent, costToParent, totalCost); - } private SpanningTree growApproach(SpanningTree spanningTree) { //this approach grows gradually the MST found in the previous step //when it is about to get larger than K, we crop the current worst leaf if the new value to be added // is actually better - + if (spanningTree.effectiveNodeCount() < k) + return spanningTree; + HugeLongArray outDegree = HugeLongArray.newArray(graph.nodeCount()); HugeLongArray parent = HugeLongArray.newArray(graph.nodeCount()); @@ -372,21 +288,6 @@ private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator m } } - private SpanningTree combineApproach(SpanningTree tree) { - if (tree.effectiveNodeCount() < k) { - return tree; - } - var spanningTree1 = cutLeafApproach(tree); - var spanningTree2 = growApproach(tree); - System.out.println("Prune Leaf: " + spanningTree1.totalWeight() + " Grow Mst:" + spanningTree2.totalWeight()); - if (spanningTree1.totalWeight() > spanningTree2.totalWeight()) { - return (minMax == Prim.MAX_OPERATOR) ? spanningTree1 : spanningTree2; - } else { - return (minMax == Prim.MAX_OPERATOR) ? spanningTree2 : spanningTree1; - - } - - } } diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java index 0a6e8748c40..70c2451b297 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeAlgorithmFactory.java @@ -53,9 +53,7 @@ public Task progressTask( return Tasks.task( taskName(), Tasks.leaf("SpanningTree", graph.relationshipCount()), - Tasks.leaf("Remove relationships 1"), - Tasks.leaf("Remove relationships 2") - + Tasks.leaf("Remove relationships") ); } diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java index bfe584ceedf..f0ecfa60b06 100644 --- a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/spanningtree/KSpanningTreeTest.java @@ -133,13 +133,13 @@ void shouldProduceSingleConnectedTree() { ", (c:Node)" + ", (d:Node)" + ", (e:Node)" + - ", (a)-[:TYPE {cost: 1.0}]->(b)" + - ", (b)-[:TYPE {cost: 20.0}]->(c)" + - ", (c)-[:TYPE {cost: 30.0}]->(d)" + + ", (b)-[:TYPE {cost: 1.0}]->(a)" + + ", (c)-[:TYPE {cost: 20.0}]->(b)" + + ", (d)-[:TYPE {cost: 30.0}]->(c)" + ", (d)-[:TYPE {cost: 1.0}]->(e)" ); var graph = factory.build().getUnion(); - var startNode = factory.nodeId("a"); + var startNode = factory.nodeId("d"); var k = 3; var spanningTree = new KSpanningTree( @@ -172,14 +172,14 @@ void shouldProduceSingleTreeWithKMinusOneEdges(int k, double expected) { ", (d:Node)" + ", (e:Node)" + ", (f:Node)" + - ", (a)-[:TYPE {cost: 1.0}]->(b)" + - ", (b)-[:TYPE {cost: 20.0}]->(c)" + - ", (c)-[:TYPE {cost: 30.0}]->(d)" + + ", (b)-[:TYPE {cost: 1.0}]->(a)" + + ", (c)-[:TYPE {cost: 20.0}]->(b)" + + ", (d)-[:TYPE {cost: 30.0}]->(c)" + ", (d)-[:TYPE {cost: 1.0}]->(e)" + ", (e)-[:TYPE {cost: 1.0}]->(f)" ); var graph = factory.build().getUnion(); - var startNode = factory.nodeId("a"); + var startNode = factory.nodeId("d"); var spanningTree = new KSpanningTree( @@ -201,52 +201,6 @@ void shouldProduceSingleTreeWithKMinusOneEdges(int k, double expected) { assertThat(spanningTree.totalWeight()).isEqualTo(expected); } - @Test - void worstCaseForCuttingHeaviest() { - var factory = GdlFactory.of("CREATE" + - " (a:Node)" + - ", (b:Node)" + - ", (c:Node)" + - ", (d:Node)" + - ", (e:Node)" + - ", (f:Node)" + - ", (g:Node)" + - ", (h:Node)" + - ", (a)-[:TYPE {cost: 9.0}]->(b)" + - ", (b)-[:TYPE {cost: 9.0}]->(c)" + - ", (c)-[:TYPE {cost: 0.0}]->(d)" + - ", (d)-[:TYPE {cost: 10.0}]->(e)" + - ", (e)-[:TYPE {cost: 0.0}]->(f)" + - ", (f)-[:TYPE {cost: 9.0}]->(g)" + - ", (g)-[:TYPE {cost: 9.0}]->(h)" - - ); - var graph = factory.build().getUnion(); - var startNode = factory.nodeId("a"); - - - var spanningTree = new KSpanningTree( - graph, - Prim.MIN_OPERATOR, - startNode, - 4, - ProgressTracker.NULL_TRACKER - ).compute(); - - var counter = new MutableLong(0); - spanningTree.forEach((__, ___, ____) -> { - counter.add(1); - return true; - }); - - assertThat(counter.getValue()).isEqualTo(4 - 1); - assertThat(spanningTree.totalWeight()).isEqualTo(10.0); - //here a 'cut-heaviest' approach would begin by cutting edge 10. - //this would prune the tree into two smaller subtrees of 4 nodes each - //adn thus return a 18 despite the optimal being 10. - //this is one of the many situations where cut-heavy-fails - - } @Test void worstCaseForPruningLeaves() { @@ -344,16 +298,12 @@ void shouldLogProgress() { "KSpanningTree :: SpanningTree 80%", "KSpanningTree :: SpanningTree 100%", "KSpanningTree :: SpanningTree :: Finished", - "KSpanningTree :: Remove relationships 1 :: Start", - "KSpanningTree :: Remove relationships 1 50%", - "KSpanningTree :: Remove relationships 1 100%", - "KSpanningTree :: Remove relationships 1 :: Finished", - "KSpanningTree :: Remove relationships 2 :: Start", - "KSpanningTree :: Remove relationships 2 20%", - "KSpanningTree :: Remove relationships 2 40%", - "KSpanningTree :: Remove relationships 2 60%", - "KSpanningTree :: Remove relationships 2 100%", - "KSpanningTree :: Remove relationships 2 :: Finished", + "KSpanningTree :: Remove relationships :: Start", + "KSpanningTree :: Remove relationships 20%", + "KSpanningTree :: Remove relationships 40%", + "KSpanningTree :: Remove relationships 60%", + "KSpanningTree :: Remove relationships 100%", + "KSpanningTree :: Remove relationships :: Finished", "KSpanningTree :: Finished" ); } From 58e34d4e7d52131ef715526efb0a4addfd4adee8 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 23 Jan 2023 11:57:17 +0100 Subject: [PATCH 056/400] Refactor KSP heuristic to make it less dense in text Co-authored-by: Lasse Westh-Nielsen --- .../gds/impl/spanningtree/KSpanningTree.java | 94 ++++++++++++------- 1 file changed, 60 insertions(+), 34 deletions(-) diff --git a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java index 19b422f8ce8..59f1eb06e62 100644 --- a/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java +++ b/alpha/alpha-algo/src/main/java/org/neo4j/gds/impl/spanningtree/KSpanningTree.java @@ -117,7 +117,7 @@ private SpanningTree growApproach(SpanningTree spanningTree) { // is actually better if (spanningTree.effectiveNodeCount() < k) return spanningTree; - + HugeLongArray outDegree = HugeLongArray.newArray(graph.nodeCount()); HugeLongArray parent = HugeLongArray.newArray(graph.nodeCount()); @@ -152,10 +152,7 @@ private SpanningTree growApproach(SpanningTree spanningTree) { nodesInTree++; nodeAdded = true; } else { - while (!exterior.get(toTrim.top())) { //not valid frontier nodes anymore, just ignore - toTrim.pop(); //as we said, pq does not have a direct remove method - } - var nodeToTrim = toTrim.top(); //a leaf node with worst cost + var nodeToTrim = findNextValidLeaf(toTrim, exterior); //a leaf node with currently theworst cost if (parent.get(node) == nodeToTrim) { //we cannot add it, if we're supposed to remove its parent //TODO: should be totally feasible to consider the 2nd worst then. @@ -197,13 +194,7 @@ private SpanningTree growApproach(SpanningTree spanningTree) { } } if (affectedNode != -1) { //if a node has been converted to a leaf - if (!toTrim.containsElement(affectedNode)) { - toTrim.add(affectedNode, affectedCost); //add it to pq - } else { - //it is still in the queue, but it is not a leaf anymore, so it's value is obsolete - toTrim.set(affectedNode, affectedCost); - } - exterior.set(affectedNode); //and mark it in the exterior + updateExterior(affectedNode, affectedCost, toTrim, exterior); } } else { //the root is removed, long live the new root! @@ -212,14 +203,9 @@ private SpanningTree growApproach(SpanningTree spanningTree) { var newRoot = rootNodeAdjacent.nextSetBit(0); rootNodeAdjacent.clear(); //empty everything //find the children of the new root (this can happen once per node) - graph.forEachRelationship(newRoot, (s, t) -> { - //relevant are only those nodes which are currently - //in the k-tree - if (parent.get(t) == s && included.get(t)) { - rootNodeAdjacent.set(t); - } - return true; - }); + + fillChildren(newRoot, rootNodeAdjacent, parent, included); + root = newRoot; //set it as root clearNode(root, parent, costToParent); @@ -247,32 +233,26 @@ private SpanningTree growApproach(SpanningTree spanningTree) { //then the node (being a leaf) is added to the trimming priority queu toTrim.add(node, associatedCost); exterior.set(node); //and the exterior + relaxNode(node, priorityQueue, parent, spanningTree); - graph.forEachRelationship(node, (s, t) -> { - if (parent.get(t) == s) { - //TODO: work's only on mst edges for now (should be doable to re-find an k-MST from whole graph) - if (!priorityQueue.containsElement(t)) { - priorityQueue.add(t, spanningTree.costToParent(t)); - } - - } - return true; - }); } else { clearNode(node, parent, costToParent); - } } //post-processing step: anything not touched is reset to -1 + pruneUntouchedNodes(parent, costToParent, included); + progressTracker.endSubTask(); + return new SpanningTree(root, graph.nodeCount(), k, parent, costToParent, totalCost); + + } + + private void pruneUntouchedNodes(HugeLongArray parent, HugeDoubleArray costToParent, BitSet included) { graph.forEachNode(nodeId -> { if (!included.get(nodeId)) { clearNode(nodeId, parent, costToParent); } return true; }); - progressTracker.endSubTask(); - return new SpanningTree(root, graph.nodeCount(), k, parent, costToParent, totalCost); - } private void clearNode(long node, HugeLongArray parent, HugeDoubleArray costToParent) { @@ -288,6 +268,52 @@ private boolean moveMakesSense(double cost1, double cost2, DoubleUnaryOperator m } } + private void updateExterior(long affectedNode, double affectedCost, HugeLongPriorityQueue toTrim, BitSet exterior) { + if (!toTrim.containsElement(affectedNode)) { + toTrim.add(affectedNode, affectedCost); //add it to pq + } else { + //it is still in the queue, but it is not a leaf anymore, so it's value is obsolete + toTrim.set(affectedNode, affectedCost); + } + exterior.set(affectedNode); //and mark it in the exterior + } + + private long findNextValidLeaf(HugeLongPriorityQueue toTrim, BitSet exterior) { + while (!exterior.get(toTrim.top())) { //not valid frontier nodes anymore, just ignore + toTrim.pop(); //as we said, pq does not have a direct remove method + } + return toTrim.top(); + } + + private void fillChildren(long newRoot, BitSet rootNodeAdjacent, HugeLongArray parent, BitSet included) { + graph.forEachRelationship(newRoot, (s, t) -> { + //relevant are only those nodes which are currently + //in the k-tree + if (parent.get(t) == s && included.get(t)) { + rootNodeAdjacent.set(t); + } + return true; + }); + } + + private void relaxNode( + long node, + HugeLongPriorityQueue priorityQueue, + HugeLongArray parent, + SpanningTree spanningTree + ) { + graph.forEachRelationship(node, (s, t) -> { + if (parent.get(t) == s) { + //TODO: work's only on mst edges for now (should be doable to re-find an k-MST from whole graph) + if (!priorityQueue.containsElement(t)) { + priorityQueue.add(t, spanningTree.costToParent(t)); + } + + } + return true; + }); + } + } From c711bf76da281bdf0d117e54d88219ac67ad5fa3 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 23 Jan 2023 12:42:59 +0100 Subject: [PATCH 057/400] Tests --- .../k-minimum-weight-spanning-tree.adoc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc index 815d331ebd3..cc72b64e573 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc @@ -129,7 +129,7 @@ By setting the `k=3`, we define that we want to find a 3-minimum spanning tree t -- [source, cypher, role=noplay] ---- -MATCH (n:Place{id: 'D'}) +MATCH (n:Place{id: 'A'}) CALL gds.alpha.kSpanningTree.write('graph', { k: 3, sourceNode: id(n), @@ -159,12 +159,12 @@ RETURN n.id As Place, p as Partition [opts="header",cols="1,1"] |=== | Place | Partition -| "A" | 1 -| "B" | 1 -| "C" | 1 +| "A" | 0 +| "B" | 0 +| "C" | 0 |=== -- -Nodes A, B, and C are the result 3-minimum spanning tree of our graph. +Nodes A, B, and C form the discovered 3-minimum spanning tree of our graph. === Maximum K-Spanning Tree example From e411ce51894cbae053951a5462c5c13bcc3f92fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Mon, 23 Jan 2023 13:25:25 +0100 Subject: [PATCH 058/400] Move Defaults & Limits docs --- doc/modules/ROOT/content-nav.adoc | 2 +- doc/modules/ROOT/pages/introduction.adoc | 2 +- doc/modules/ROOT/pages/management-ops/index.adoc | 1 - .../additional-operation-references.adoc | 8 ++++---- .../defaults-and-limits.adoc | 0 doc/modules/ROOT/pages/production-deployment/index.adoc | 1 + 6 files changed, 7 insertions(+), 7 deletions(-) rename doc/modules/ROOT/pages/{management-ops => production-deployment}/defaults-and-limits.adoc (100%) diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 66be7da9f18..34790329a1e 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -38,7 +38,6 @@ ** xref:management-ops/create-cypher-db.adoc[] ** xref:management-ops/administration.adoc[] ** xref:management-ops/backup-restore.adoc[] -** xref:management-ops/defaults-and-limits.adoc[] * xref:algorithms/index.adoc[] ** xref:algorithms/syntax.adoc[] ** xref:algorithms/centrality.adoc[] @@ -145,6 +144,7 @@ * xref:end-to-end-examples/end-to-end-examples.adoc[] ** xref:end-to-end-examples/fastrp-knn-example.adoc[] * xref:production-deployment/index.adoc[] +** xref:production-deployment/defaults-and-limits.adoc[] ** xref:production-deployment/transaction-handling.adoc[] ** xref:production-deployment/composite.adoc[] ** xref:production-deployment/neo4j-cluster.adoc[] diff --git a/doc/modules/ROOT/pages/introduction.adoc b/doc/modules/ROOT/pages/introduction.adoc index f59bcf7a26a..1455bba2af5 100644 --- a/doc/modules/ROOT/pages/introduction.adoc +++ b/doc/modules/ROOT/pages/introduction.adoc @@ -107,6 +107,6 @@ The Neo4j Graph Data Science library is available in two editions. *** Sharing of models between users, by xref:model-catalog/publish.adoc[publishing it]. *** Model xref:model-catalog/store.adoc#model-catalog-store-ops[persistence to disk]. ** Supports an xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[optimized graph implementation]. -** Allows the configuration of xref:management-ops/defaults-and-limits.adoc[defaults and limits]. +** Allows the configuration of xref:production-deployment/defaults-and-limits.adoc[defaults and limits]. For more information see xref:installation/System-requirements.adoc#system-requirements-cpu[System Requirements - CPU]. diff --git a/doc/modules/ROOT/pages/management-ops/index.adoc b/doc/modules/ROOT/pages/management-ops/index.adoc index b2645bdcf69..bb2c4a3bb63 100644 --- a/doc/modules/ROOT/pages/management-ops/index.adoc +++ b/doc/modules/ROOT/pages/management-ops/index.adoc @@ -13,4 +13,3 @@ This chapter is divided into the following sections: * xref:management-ops/create-cypher-db.adoc[Cypher on GDS graph] * xref:management-ops/administration.adoc[Administration] * xref:management-ops/backup-restore.adoc[Backup and Restore] -* xref:management-ops/defaults-and-limits.adoc[Defaults and Limits] diff --git a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc index 55ec88482d4..b9d3e745dac 100644 --- a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc @@ -25,8 +25,8 @@ | xref:common-usage/monitoring-system.adoc[Get an overview of the system's workload and available resources] | `gds.alpha.systemMonitor` | Back-up graphs and models to disk | `gds.alpha.backup` | Restore persisted graphs and models to memory | `gds.alpha.restore` -| xref:management-ops/defaults-and-limits.adoc[List configured defaults] | `gds.alpha.config.defaults.list` -| xref:management-ops/defaults-and-limits.adoc[Configure a default] | `gds.alpha.config.defaults.set` -| xref:management-ops/defaults-and-limits.adoc#_limits_on_configuration_values[List configured limits] | `gds.alpha.config.limits.list` -| xref:management-ops/defaults-and-limits.adoc#_limits_on_configuration_values[Configure a limit] | `gds.alpha.config.limits.set` +| xref:production-deployment/defaults-and-limits.adoc[List configured defaults] | `gds.alpha.config.defaults.list` +| xref:production-deployment/defaults-and-limits.adoc[Configure a default] | `gds.alpha.config.defaults.set` +| xref:production-deployment/defaults-and-limits.adoc#_limits_on_configuration_values[List configured limits] | `gds.alpha.config.limits.list` +| xref:production-deployment/defaults-and-limits.adoc#_limits_on_configuration_values[Configure a limit] | `gds.alpha.config.limits.set` |=== diff --git a/doc/modules/ROOT/pages/management-ops/defaults-and-limits.adoc b/doc/modules/ROOT/pages/production-deployment/defaults-and-limits.adoc similarity index 100% rename from doc/modules/ROOT/pages/management-ops/defaults-and-limits.adoc rename to doc/modules/ROOT/pages/production-deployment/defaults-and-limits.adoc diff --git a/doc/modules/ROOT/pages/production-deployment/index.adoc b/doc/modules/ROOT/pages/production-deployment/index.adoc index 44959c780b0..2ebd38af2ac 100644 --- a/doc/modules/ROOT/pages/production-deployment/index.adoc +++ b/doc/modules/ROOT/pages/production-deployment/index.adoc @@ -5,6 +5,7 @@ This chapter is divided into the following sections: +* xref:production-deployment/defaults-and-limits.adoc[Defaults and Limits] * xref:production-deployment/transaction-handling.adoc[Transaction Handling] * xref:production-deployment/composite.adoc[Using GDS and Composite databases] * xref:production-deployment/neo4j-cluster.adoc[GDS with Neo4j cluster] From 620ad77d535c8a70d700d274057d7b893d6bdf17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Fri, 20 Jan 2023 13:36:42 +0100 Subject: [PATCH 059/400] Use computation instead of RelationshipWeightApplier --- .../gds/beta/pregel/ForkJoinComputer.java | 4 ++-- .../gds/beta/pregel/PartitionedComputer.java | 4 ++-- .../beta/pregel/context/ComputeContext.java | 20 ++++++++----------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java index 6f480a62300..b8509932b7f 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java @@ -103,7 +103,7 @@ void release() { Supplier> computeContext = () -> new ComputeContext<>( graph.concurrentCopy(), config, - ((PregelComputation) computation)::applyRelationshipWeight, + computation, nodeValues, messenger, voteBits, @@ -144,7 +144,7 @@ void release() { Supplier> computeContext = () -> new BidirectionalComputeContext<>( graph.concurrentCopy(), config, - ((BidirectionalPregelComputation) computation)::applyRelationshipWeight, + computation, nodeValues, messenger, voteBits, diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java index e5285a8e2b4..8311f240149 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java @@ -149,7 +149,7 @@ void release() { var computeContext = new ComputeContext<>( graph, config, - ((PregelComputation) computation)::applyRelationshipWeight, + computation, nodeValues, messenger, voteBits, @@ -192,7 +192,7 @@ void release() { var computeContext = new BidirectionalComputeContext<>( graph, config, - ((BidirectionalPregelComputation) computation)::applyRelationshipWeight, + computation, nodeValues, messenger, voteBits, diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java index 380166b9af3..d556641a040 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.beta.pregel.BasePregelComputation; import org.neo4j.gds.beta.pregel.Messenger; import org.neo4j.gds.beta.pregel.NodeValue; import org.neo4j.gds.beta.pregel.PregelConfig; @@ -36,16 +37,16 @@ */ public class ComputeContext extends NodeCentricContext { - final RelationshipWeightApplier relationshipWeightApplier; private final HugeAtomicBitSet voteBits; private final Messenger messenger; private final MutableInt iteration; private final AtomicBoolean hasSendMessage; + protected BasePregelComputation computation; public ComputeContext(Graph graph, CONFIG config, - RelationshipWeightApplier relationshipWeightApplier, + BasePregelComputation computation, NodeValue nodeValue, Messenger messenger, HugeAtomicBitSet voteBits, @@ -53,7 +54,7 @@ public ComputeContext(Graph graph, AtomicBoolean hasSendMessage, ProgressTracker progressTracker) { super(graph, config, nodeValue, progressTracker); - this.relationshipWeightApplier = relationshipWeightApplier; + this.computation = computation; this.sendMessagesFunction = config.hasRelationshipWeightProperty() ? this::sendToNeighborsWeighted : this::sendToNeighbors; @@ -163,7 +164,7 @@ private void sendToNeighbors(long sourceNodeId, double message) { private void sendToNeighborsWeighted(long sourceNodeId, double message) { graph.forEachRelationship(sourceNodeId, 1.0, (ignored, targetNodeId, weight) -> { - sendTo(targetNodeId, relationshipWeightApplier.applyRelationshipWeight(message, weight)); + sendTo(targetNodeId, computation.applyRelationshipWeight(message, weight)); return true; }); } @@ -173,11 +174,6 @@ interface SendMessagesFunction { void sendToNeighbors(long sourceNodeId, double message); } - @FunctionalInterface - public interface RelationshipWeightApplier { - double applyRelationshipWeight(double nodeValue, double relationshipWeight); - } - public static final class BidirectionalComputeContext extends ComputeContext implements BidirectionalNodeCentricContext { private final SendMessagesIncomingFunction sendMessagesIncomingFunction; @@ -185,7 +181,7 @@ public static final class BidirectionalComputeContext computation, NodeValue nodeValue, Messenger messenger, HugeAtomicBitSet voteBits, @@ -196,7 +192,7 @@ public BidirectionalComputeContext( super( graph, config, - relationshipWeightApplier, + computation, nodeValue, messenger, voteBits, @@ -226,7 +222,7 @@ private void sendToIncomingNeighbors(long sourceNodeId, double message) { private void sendToIncomingNeighborsWeighted(long sourceNodeId, double message) { graph.forEachInverseRelationship(sourceNodeId, 1.0, (ignored, targetNodeId, weight) -> { - sendTo(targetNodeId, relationshipWeightApplier.applyRelationshipWeight(message, weight)); + sendTo(targetNodeId, computation.applyRelationshipWeight(message, weight)); return true; }); } From 2e2545f0a4998a4bef5da1b66f98605613eaedbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Fri, 20 Jan 2023 16:05:35 +0100 Subject: [PATCH 060/400] Remove contention on hasSendMessage atomic boolean --- .../gds/beta/pregel/ForkJoinComputeStep.java | 23 +++++++++++-------- .../gds/beta/pregel/ForkJoinComputer.java | 2 -- .../beta/pregel/PartitionedComputeStep.java | 1 + .../gds/beta/pregel/PartitionedComputer.java | 2 -- .../beta/pregel/context/ComputeContext.java | 16 ++++++------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java index fd8b35d720c..a3d2669bb78 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java @@ -43,8 +43,9 @@ public final class ForkJoinComputeStep< private final InitFunction initFunction; private final ComputeFunction computeFunction; - private final Supplier initContext; - private final Supplier computeContext; + private final Supplier initContextSupplier; + private final Supplier computeContextSupplier; + private final COMPUTE_CONTEXT computeContext; private final NodeValue nodeValue; private final HugeAtomicBitSet voteBits; private final Messenger messenger; @@ -56,8 +57,8 @@ public final class ForkJoinComputeStep< ForkJoinComputeStep( InitFunction initFunction, ComputeFunction computeFunction, - Supplier initContext, - Supplier computeContext, + Supplier initContextSupplier, + Supplier computeContextSupplier, MutableInt iteration, Partition nodeBatch, NodeValue nodeValue, @@ -70,8 +71,8 @@ public final class ForkJoinComputeStep< super(parent); this.initFunction = initFunction; this.computeFunction = computeFunction; - this.initContext = initContext; - this.computeContext = computeContext; + this.initContextSupplier = initContextSupplier; + this.computeContextSupplier = computeContextSupplier; this.iteration = iteration; this.voteBits = voteBits; this.nodeBatch = nodeBatch; @@ -79,6 +80,7 @@ public final class ForkJoinComputeStep< this.messenger = messenger; this.hasSentMessage = sentMessage; this.progressTracker = progressTracker; + this.computeContext = computeContextSupplier.get(); } @Override @@ -99,8 +101,8 @@ public void compute() { var leftTask = new ForkJoinComputeStep<>( initFunction, computeFunction, - initContext, - computeContext, + initContextSupplier, + computeContextSupplier, iteration, leftBatch, nodeValue, @@ -119,6 +121,7 @@ public void compute() { this.compute(); } else { computeBatch(); + hasSentMessage.set(computeContext.hasSentMessage()); tryComplete(); } } @@ -155,12 +158,12 @@ public Partition nodeBatch() { @Override public INIT_CONTEXT initContext() { - return initContext.get(); + return initContextSupplier.get(); } @Override public COMPUTE_CONTEXT computeContext() { - return computeContext.get(); + return computeContext; } @Override diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java index b8509932b7f..a94bffebb29 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java @@ -108,7 +108,6 @@ void release() { messenger, voteBits, iteration, - hasSentMessages, progressTracker ); @@ -149,7 +148,6 @@ void release() { messenger, voteBits, iteration, - hasSentMessages, progressTracker ); diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java index e8b242d2ab5..9ef61f155c4 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java @@ -78,6 +78,7 @@ public final class PartitionedComputeStep< @Override public void run() { computeBatch(); + hasSentMessage.set(computeContext().hasSentMessage()); } @Override diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java index 8311f240149..bf6d53f4ba8 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java @@ -154,7 +154,6 @@ void release() { messenger, voteBits, iteration, - hasSentMessages, progressTracker ); @@ -197,7 +196,6 @@ void release() { messenger, voteBits, iteration, - hasSentMessages, progressTracker ); diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java index d556641a040..e729d058ddf 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java @@ -28,8 +28,6 @@ import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; -import java.util.concurrent.atomic.AtomicBoolean; - /** * A context that is used during the computation. It allows an implementation * to send messages to other nodes and change the state of the currently @@ -41,7 +39,8 @@ public class ComputeContext extends NodeCentricCont private final Messenger messenger; private final MutableInt iteration; - private final AtomicBoolean hasSendMessage; + private boolean hasSendMessage; + protected BasePregelComputation computation; public ComputeContext(Graph graph, @@ -51,7 +50,6 @@ public ComputeContext(Graph graph, Messenger messenger, HugeAtomicBitSet voteBits, MutableInt iteration, - AtomicBoolean hasSendMessage, ProgressTracker progressTracker) { super(graph, config, nodeValue, progressTracker); this.computation = computation; @@ -61,7 +59,7 @@ public ComputeContext(Graph graph, this.messenger = messenger; this.voteBits = voteBits; this.iteration = iteration; - this.hasSendMessage = hasSendMessage; + this.hasSendMessage = false; } private final SendMessagesFunction sendMessagesFunction; @@ -152,7 +150,7 @@ public void sendToNeighbors(double message) { */ public void sendTo(long targetNodeId, double message) { messenger.sendTo(targetNodeId, message); - this.hasSendMessage.set(true); + this.hasSendMessage = true; } private void sendToNeighbors(long sourceNodeId, double message) { @@ -169,6 +167,10 @@ private void sendToNeighborsWeighted(long sourceNodeId, double message) { }); } + public boolean hasSentMessage() { + return hasSendMessage; + } + @FunctionalInterface interface SendMessagesFunction { void sendToNeighbors(long sourceNodeId, double message); @@ -186,7 +188,6 @@ public BidirectionalComputeContext( Messenger messenger, HugeAtomicBitSet voteBits, MutableInt iteration, - AtomicBoolean hasSendMessage, ProgressTracker progressTracker ) { super( @@ -197,7 +198,6 @@ public BidirectionalComputeContext( messenger, voteBits, iteration, - hasSendMessage, progressTracker ); From ff797d0dfcdb2ff20b9c90cc4a32d889a072d234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Fri, 20 Jan 2023 17:59:54 +0100 Subject: [PATCH 061/400] Fix check for sendMessages for PartitionedComputer --- .../neo4j/gds/beta/pregel/ForkJoinComputeStep.java | 2 +- .../neo4j/gds/beta/pregel/ForkJoinComputer.java | 3 +++ .../gds/beta/pregel/PartitionedComputeStep.java | 12 +++++------- .../neo4j/gds/beta/pregel/PartitionedComputer.java | 8 +++++--- .../gds/beta/pregel/context/ComputeContext.java | 14 ++++++++++---- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java index a3d2669bb78..87d107d451d 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputeStep.java @@ -121,7 +121,7 @@ public void compute() { this.compute(); } else { computeBatch(); - hasSentMessage.set(computeContext.hasSentMessage()); + hasSentMessage.compareAndSet(false, computeContext.hasSentMessage()); tryComplete(); } } diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java index a94bffebb29..096c40e2953 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/ForkJoinComputer.java @@ -30,6 +30,7 @@ import org.neo4j.gds.core.utils.partition.Partition; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import java.util.Optional; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; @@ -108,6 +109,7 @@ void release() { messenger, voteBits, iteration, + Optional.empty(), progressTracker ); @@ -148,6 +150,7 @@ void release() { messenger, voteBits, iteration, + Optional.empty(), progressTracker ); diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java index 9ef61f155c4..8e195f4fa42 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputeStep.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.beta.pregel; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.beta.pregel.context.ComputeContext; import org.neo4j.gds.beta.pregel.context.InitContext; @@ -26,8 +27,6 @@ import org.neo4j.gds.core.utils.partition.Partition; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; -import java.util.concurrent.atomic.AtomicBoolean; - public final class PartitionedComputeStep< CONFIG extends PregelConfig, ITERATOR extends Messages.MessageIterator, @@ -45,7 +44,7 @@ public final class PartitionedComputeStep< private final Messenger messenger; private final MutableInt iteration; - private final AtomicBoolean hasSentMessage; + private final MutableBoolean hasSentMessage; private final NodeValue nodeValue; PartitionedComputeStep( @@ -58,7 +57,7 @@ public final class PartitionedComputeStep< Messenger messenger, HugeAtomicBitSet voteBits, MutableInt iteration, - AtomicBoolean hasSentMessage, + MutableBoolean hasSentMessage, ProgressTracker progressTracker ) { this.initFunction = initFunction; @@ -78,7 +77,6 @@ public final class PartitionedComputeStep< @Override public void run() { computeBatch(); - hasSentMessage.set(computeContext().hasSentMessage()); } @Override @@ -128,10 +126,10 @@ public ProgressTracker progressTracker() { void init(int iteration) { this.iteration.setValue(iteration); - this.hasSentMessage.set(false); + hasSentMessage.setValue(false); } boolean hasSentMessage() { - return hasSentMessage.get(); + return hasSentMessage.getValue(); } } diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java index bf6d53f4ba8..c0b0da73cb2 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/PartitionedComputer.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.beta.pregel; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableInt; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.api.Graph; @@ -35,7 +36,6 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -137,7 +137,7 @@ void release() { Partition partition ) { MutableInt iteration = new MutableInt(0); - var hasSentMessages = new AtomicBoolean(false); + var hasSentMessages = new MutableBoolean(false); var initContext = new InitContext<>( graph, @@ -154,6 +154,7 @@ void release() { messenger, voteBits, iteration, + Optional.of(hasSentMessages), progressTracker ); @@ -179,7 +180,7 @@ void release() { Partition partition ) { MutableInt iteration = new MutableInt(0); - var hasSentMessages = new AtomicBoolean(false); + var hasSentMessages = new MutableBoolean(false); var initContext = new BidirectionalInitContext<>( graph, @@ -196,6 +197,7 @@ void release() { messenger, voteBits, iteration, + Optional.of(hasSentMessages), progressTracker ); diff --git a/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java b/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java index e729d058ddf..57064361d92 100644 --- a/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java +++ b/pregel/src/main/java/org/neo4j/gds/beta/pregel/context/ComputeContext.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.beta.pregel.context; +import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.api.Graph; import org.neo4j.gds.beta.pregel.BasePregelComputation; @@ -28,6 +29,8 @@ import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import java.util.Optional; + /** * A context that is used during the computation. It allows an implementation * to send messages to other nodes and change the state of the currently @@ -39,7 +42,7 @@ public class ComputeContext extends NodeCentricCont private final Messenger messenger; private final MutableInt iteration; - private boolean hasSendMessage; + private final MutableBoolean hasSendMessage; protected BasePregelComputation computation; @@ -50,6 +53,7 @@ public ComputeContext(Graph graph, Messenger messenger, HugeAtomicBitSet voteBits, MutableInt iteration, + Optional hasSendMessage, ProgressTracker progressTracker) { super(graph, config, nodeValue, progressTracker); this.computation = computation; @@ -59,7 +63,7 @@ public ComputeContext(Graph graph, this.messenger = messenger; this.voteBits = voteBits; this.iteration = iteration; - this.hasSendMessage = false; + this.hasSendMessage = hasSendMessage.orElse(new MutableBoolean(false)); } private final SendMessagesFunction sendMessagesFunction; @@ -150,7 +154,7 @@ public void sendToNeighbors(double message) { */ public void sendTo(long targetNodeId, double message) { messenger.sendTo(targetNodeId, message); - this.hasSendMessage = true; + this.hasSendMessage.setValue(true); } private void sendToNeighbors(long sourceNodeId, double message) { @@ -168,7 +172,7 @@ private void sendToNeighborsWeighted(long sourceNodeId, double message) { } public boolean hasSentMessage() { - return hasSendMessage; + return hasSendMessage.getValue(); } @FunctionalInterface @@ -188,6 +192,7 @@ public BidirectionalComputeContext( Messenger messenger, HugeAtomicBitSet voteBits, MutableInt iteration, + Optional hasSendMessage, ProgressTracker progressTracker ) { super( @@ -198,6 +203,7 @@ public BidirectionalComputeContext( messenger, voteBits, iteration, + hasSendMessage, progressTracker ); From 250659e289101b03d3e2413d6d1933434cf80361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Mon, 16 Jan 2023 16:44:25 +0100 Subject: [PATCH 062/400] Refactor node label token lookup in NodesBuilder --- .../loading/construction/GraphFactory.java | 6 +- .../loading/construction/NodesBuilder.java | 58 ++------- .../construction/TokenToNodeLabelMap.java | 110 ++++++++++++++++++ 3 files changed, 120 insertions(+), 54 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index d496997e5a9..5bebfeb9186 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -119,8 +119,7 @@ static NodesBuilder nodesBuilder( )).orElseGet(() -> new NodesBuilder( maxOriginalId, threadCount, - new ObjectIntScatterMap<>(), - new IntObjectHashMap<>(), + TokenToNodeLabelMap.lazy(), new ConcurrentHashMap<>(), idMapBuilder, labelInformation, @@ -159,8 +158,7 @@ private static NodesBuilder fromSchema( return new NodesBuilder( maxOriginalId, concurrency, - elementIdentifierLabelTokenMapping, - labelTokenNodeLabelMapping, + TokenToNodeLabelMap.fixed(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping), new ConcurrentHashMap<>(propertyBuildersByPropertyKey), idMapBuilder, hasLabelInformation, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 7d156862cc3..b2fe83a4255 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -19,8 +19,6 @@ */ package org.neo4j.gds.core.loading.construction; -import com.carrotsearch.hppc.IntObjectHashMap; -import com.carrotsearch.hppc.ObjectIntMap; import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.ValueClass; @@ -50,20 +48,15 @@ import org.neo4j.values.virtual.MapValue; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.LongAdder; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.function.LongPredicate; import static java.util.stream.Collectors.toMap; -import static org.neo4j.gds.core.GraphDimensions.NO_SUCH_LABEL; -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public final class NodesBuilder { @@ -73,27 +66,21 @@ public final class NodesBuilder { private final long maxOriginalId; private final int concurrency; - private int nextLabelId; - private final ObjectIntMap elementIdentifierLabelTokenMapping; private final IdMapBuilder idMapBuilder; private final LabelInformation.Builder labelInformationBuilder; - private final IntObjectHashMap> labelTokenNodeLabelMapping; private final LongAdder importedNodes; private final AutoCloseableThreadLocal threadLocalBuilder; private final NodeImporter nodeImporter; - private final Lock lock; - private final ConcurrentMap propertyBuildersByPropertyKey; private final boolean hasProperties; NodesBuilder( long maxOriginalId, int concurrency, - ObjectIntMap elementIdentifierLabelTokenMapping, - IntObjectHashMap> labelTokenNodeLabelMapping, + TokenToNodeLabelMap tokenToNodeLabelMap, ConcurrentMap propertyBuildersByPropertyKey, IdMapBuilder idMapBuilder, boolean hasLabelInformation, @@ -102,27 +89,20 @@ public final class NodesBuilder { ) { this.maxOriginalId = maxOriginalId; this.concurrency = concurrency; - this.elementIdentifierLabelTokenMapping = elementIdentifierLabelTokenMapping; this.idMapBuilder = idMapBuilder; this.labelInformationBuilder = !hasLabelInformation ? LabelInformationBuilders.allNodes() : LabelInformationBuilders.multiLabelWithCapacity(maxOriginalId + 1); - this.labelTokenNodeLabelMapping = labelTokenNodeLabelMapping; - this.nextLabelId = 0; - this.lock = new ReentrantLock(true); this.propertyBuildersByPropertyKey = propertyBuildersByPropertyKey; this.hasProperties = hasProperties; this.importedNodes = new LongAdder(); this.nodeImporter = new NodeImporter( idMapBuilder, labelInformationBuilder, - labelTokenNodeLabelMapping, + tokenToNodeLabelMap.labelTokenNodeLabelMapping(), hasProperties ); - Function labelTokenIdFn = elementIdentifierLabelTokenMapping.isEmpty() - ? this::getOrCreateLabelTokenId - : this::getLabelTokenId; Function propertyBuilderFn = propertyBuildersByPropertyKey.isEmpty() ? this::getOrCreatePropertyBuilder : this::getPropertyBuilder; @@ -138,7 +118,7 @@ public final class NodesBuilder { seenNodeIdPredicate, hasLabelInformation, hasProperties, - labelTokenIdFn, + tokenToNodeLabelMap, propertyBuilderFn ) ); @@ -253,28 +233,6 @@ public void close(RuntimeException exception) { throw exception; } - private int getOrCreateLabelTokenId(NodeLabel nodeLabel) { - var token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); - if (token == NO_SUCH_LABEL) { - lock.lock(); - token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); - if (token == NO_SUCH_LABEL) { - token = nextLabelId++; - labelTokenNodeLabelMapping.put(token, Collections.singletonList(nodeLabel)); - elementIdentifierLabelTokenMapping.put(nodeLabel, token); - } - lock.unlock(); - } - return token; - } - - private int getLabelTokenId(NodeLabel nodeLabel) { - if (!elementIdentifierLabelTokenMapping.containsKey(nodeLabel)) { - throw new IllegalArgumentException(formatWithLocale("No token was specified for node label %s", nodeLabel)); - } - return elementIdentifierLabelTokenMapping.get(nodeLabel); - } - private NodePropertiesFromStoreBuilder getOrCreatePropertyBuilder(String propertyKey) { return propertyBuildersByPropertyKey.computeIfAbsent( propertyKey, @@ -300,8 +258,8 @@ private static class ThreadLocalBuilder implements AutoCloseable { private final LongAdder importedNodes; private final LongPredicate seenNodeIdPredicate; + private final TokenToNodeLabelMap tokenToNodeLabelMap; private final NodesBatchBuffer buffer; - private final Function labelTokenIdFn; private final Function propertyBuilderFn; private final NodeImporter nodeImporter; private final List batchNodeProperties; @@ -313,12 +271,12 @@ private static class ThreadLocalBuilder implements AutoCloseable { LongPredicate seenNodeIdPredicate, boolean hasLabelInformation, boolean hasProperties, - Function labelTokenIdFn, + TokenToNodeLabelMap tokenToNodeLabelMap, Function propertyBuilderFn ) { this.importedNodes = importedNodes; this.seenNodeIdPredicate = seenNodeIdPredicate; - this.labelTokenIdFn = labelTokenIdFn; + this.tokenToNodeLabelMap = tokenToNodeLabelMap; this.propertyBuilderFn = propertyBuilderFn; this.buffer = new NodesBatchBufferBuilder() @@ -365,7 +323,7 @@ private long[] labelTokens(NodeLabelToken nodeLabels) { long[] labelIds = new long[nodeLabels.size()]; for (int i = 0; i < labelIds.length; i++) { - labelIds[i] = labelTokenIdFn.apply(nodeLabels.get(i)); + labelIds[i] = tokenToNodeLabelMap.getTokenForNodeNodeLabel(nodeLabels.get(i)); } return labelIds; @@ -421,7 +379,7 @@ private int importProperty(long neoNodeId, String propertyKey, Value value) { private long[] anyLabelArray() { var anyLabelArray = this.anyLabelArray; if (anyLabelArray[0] == NOT_INITIALIZED) { - anyLabelArray[0] = labelTokenIdFn.apply(NodeLabel.ALL_NODES); + anyLabelArray[0] = tokenToNodeLabelMap.getTokenForNodeNodeLabel(NodeLabel.ALL_NODES); } return anyLabelArray; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java new file mode 100644 index 00000000000..f57631e7eb9 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import com.carrotsearch.hppc.IntObjectHashMap; +import com.carrotsearch.hppc.ObjectIntMap; +import com.carrotsearch.hppc.ObjectIntScatterMap; +import org.neo4j.gds.NodeLabel; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static org.neo4j.gds.core.GraphDimensions.NO_SUCH_LABEL; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public abstract class TokenToNodeLabelMap { + + final ObjectIntMap elementIdentifierLabelTokenMapping; + final IntObjectHashMap> labelTokenNodeLabelMapping; + + public static TokenToNodeLabelMap fixed( + ObjectIntMap elementIdentifierLabelTokenMapping, + IntObjectHashMap> labelTokenNodeLabelMapping + ) { + return new FixedTokenToNodeLabelMap(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); + } + + public static TokenToNodeLabelMap lazy() { + return new LazyTokenToNodeLabelMap(); + } + + TokenToNodeLabelMap( + ObjectIntMap elementIdentifierLabelTokenMapping, + IntObjectHashMap> labelTokenNodeLabelMapping + ) { + this.elementIdentifierLabelTokenMapping = elementIdentifierLabelTokenMapping; + this.labelTokenNodeLabelMapping = labelTokenNodeLabelMapping; + } + + IntObjectHashMap> labelTokenNodeLabelMapping() { + return this.labelTokenNodeLabelMapping; + } + + public abstract int getTokenForNodeNodeLabel(NodeLabel nodeLabel); + + static class FixedTokenToNodeLabelMap extends TokenToNodeLabelMap { + + FixedTokenToNodeLabelMap( + ObjectIntMap elementIdentifierLabelTokenMapping, + IntObjectHashMap> labelTokenNodeLabelMapping + ) { + super(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); + } + + @Override + public int getTokenForNodeNodeLabel(NodeLabel nodeLabel) { + if (!elementIdentifierLabelTokenMapping.containsKey(nodeLabel)) { + throw new IllegalArgumentException(formatWithLocale("No token was specified for node label %s", nodeLabel)); + } + return elementIdentifierLabelTokenMapping.get(nodeLabel); + } + } + + static class LazyTokenToNodeLabelMap extends TokenToNodeLabelMap { + + private final Lock lock; + private int nextLabelId; + + LazyTokenToNodeLabelMap() { + super(new ObjectIntScatterMap<>(), new IntObjectHashMap<>()); + this.lock = new ReentrantLock(true); + this.nextLabelId = 0; + } + + @Override + public int getTokenForNodeNodeLabel(NodeLabel nodeLabel) { + var token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); + if (token == NO_SUCH_LABEL) { + lock.lock(); + token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); + if (token == NO_SUCH_LABEL) { + token = nextLabelId++; + labelTokenNodeLabelMapping.put(token, Collections.singletonList(nodeLabel)); + elementIdentifierLabelTokenMapping.put(nodeLabel, token); + } + lock.unlock(); + } + return token; + } + } +} From fc151e88155d5163e668e44883b9e340f347d5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 18 Jan 2023 10:28:02 +0100 Subject: [PATCH 063/400] Fix typo in method name Co-authored-by: Martin Junghanns --- .../neo4j/gds/core/loading/construction/NodesBuilder.java | 4 ++-- .../gds/core/loading/construction/TokenToNodeLabelMap.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index b2fe83a4255..f00c3d32496 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -323,7 +323,7 @@ private long[] labelTokens(NodeLabelToken nodeLabels) { long[] labelIds = new long[nodeLabels.size()]; for (int i = 0; i < labelIds.length; i++) { - labelIds[i] = tokenToNodeLabelMap.getTokenForNodeNodeLabel(nodeLabels.get(i)); + labelIds[i] = tokenToNodeLabelMap.getTokenForNodeLabel(nodeLabels.get(i)); } return labelIds; @@ -379,7 +379,7 @@ private int importProperty(long neoNodeId, String propertyKey, Value value) { private long[] anyLabelArray() { var anyLabelArray = this.anyLabelArray; if (anyLabelArray[0] == NOT_INITIALIZED) { - anyLabelArray[0] = tokenToNodeLabelMap.getTokenForNodeNodeLabel(NodeLabel.ALL_NODES); + anyLabelArray[0] = tokenToNodeLabelMap.getTokenForNodeLabel(NodeLabel.ALL_NODES); } return anyLabelArray; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java index f57631e7eb9..672f58aa7a6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java @@ -60,7 +60,7 @@ IntObjectHashMap> labelTokenNodeLabelMapping() { return this.labelTokenNodeLabelMapping; } - public abstract int getTokenForNodeNodeLabel(NodeLabel nodeLabel); + public abstract int getTokenForNodeLabel(NodeLabel nodeLabel); static class FixedTokenToNodeLabelMap extends TokenToNodeLabelMap { @@ -72,7 +72,7 @@ static class FixedTokenToNodeLabelMap extends TokenToNodeLabelMap { } @Override - public int getTokenForNodeNodeLabel(NodeLabel nodeLabel) { + public int getTokenForNodeLabel(NodeLabel nodeLabel) { if (!elementIdentifierLabelTokenMapping.containsKey(nodeLabel)) { throw new IllegalArgumentException(formatWithLocale("No token was specified for node label %s", nodeLabel)); } @@ -92,7 +92,7 @@ static class LazyTokenToNodeLabelMap extends TokenToNodeLabelMap { } @Override - public int getTokenForNodeNodeLabel(NodeLabel nodeLabel) { + public int getTokenForNodeLabel(NodeLabel nodeLabel) { var token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); if (token == NO_SUCH_LABEL) { lock.lock(); From 2ac511891158cc64167c9d5b177ce08d9ff4906d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Mon, 16 Jan 2023 17:38:49 +0100 Subject: [PATCH 064/400] Add property state override to NodesBuilder --- .../gds/core/loading/construction/GraphFactory.java | 9 ++++++--- .../gds/core/loading/construction/NodesBuilder.java | 13 +++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 5bebfeb9186..85ea165feea 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -89,7 +89,8 @@ static NodesBuilder nodesBuilder( Optional hasLabelInformation, Optional hasProperties, Optional deduplicateIds, - Optional concurrency + Optional concurrency, + Optional propertyStateOverride ) { boolean labelInformation = nodeSchema .map(schema -> !(schema.availableLabels().isEmpty() && schema.containsOnlyAllNodesLabel())) @@ -124,7 +125,8 @@ static NodesBuilder nodesBuilder( idMapBuilder, labelInformation, hasProperties.orElse(false), - deduplicate + deduplicate, + __ -> propertyStateOverride.orElse(PropertyState.PERSISTENT) )); } @@ -163,7 +165,8 @@ private static NodesBuilder fromSchema( idMapBuilder, hasLabelInformation, nodeSchema.hasProperties(), - deduplicateIds + deduplicateIds, + propertyKey -> nodeSchema.unionProperties().get(propertyKey).state() ); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index f00c3d32496..22ee871fdc8 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -24,6 +24,7 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; @@ -67,6 +68,7 @@ public final class NodesBuilder { private final int concurrency; private final IdMapBuilder idMapBuilder; + private final Function propertyStates; private final LabelInformation.Builder labelInformationBuilder; private final LongAdder importedNodes; @@ -85,11 +87,13 @@ public final class NodesBuilder { IdMapBuilder idMapBuilder, boolean hasLabelInformation, boolean hasProperties, - boolean deduplicateIds + boolean deduplicateIds, + Function propertyStates ) { this.maxOriginalId = maxOriginalId; this.concurrency = concurrency; this.idMapBuilder = idMapBuilder; + this.propertyStates = propertyStates; this.labelInformationBuilder = !hasLabelInformation ? LabelInformationBuilders.allNodes() : LabelInformationBuilders.multiLabelWithCapacity(maxOriginalId + 1); @@ -208,15 +212,16 @@ public Nodes build(long highestNeoId) { private Map buildProperties(IdMap idMap) { return propertyBuildersByPropertyKey.entrySet().stream().collect(toMap( Map.Entry::getKey, - entry -> entryToNodeProperty(entry, idMap) + entry -> entryToNodeProperty(entry, propertyStates.apply(entry.getKey()), idMap) )); } - private static NodeProperty entryToNodeProperty(Map.Entry entry, IdMap idMap) { + private static NodeProperty entryToNodeProperty(Map.Entry entry, PropertyState propertyState, IdMap idMap) { var nodePropertyValues = entry.getValue().build(idMap); + var valueType = nodePropertyValues.valueType(); return ImmutableNodeProperty.builder() .values(nodePropertyValues) - .propertySchema(PropertySchema.of(entry.getKey(), nodePropertyValues.valueType())) + .propertySchema(PropertySchema.of(entry.getKey(), valueType, valueType.fallbackValue(), propertyState)) .build(); } From 0c8535ac606e114bc375816653999ebaa4f36a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Mon, 16 Jan 2023 17:45:08 +0100 Subject: [PATCH 065/400] Store transient properties for Cypher loading --- .../gds/core/loading/CypherNodeLoader.java | 24 ++----------------- .../neo4j/gds/projection/GraphAggregator.java | 9 ++++++- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java index 1614190ff96..ae95c001f01 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java @@ -20,19 +20,15 @@ package org.neo4j.gds.core.loading; import org.immutables.value.Value; -import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodesBuilder; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.kernel.impl.coreapi.InternalTransaction; -import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; @Value.Enclosing class CypherNodeLoader extends CypherRecordLoader { @@ -71,6 +67,7 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { .maxOriginalId(NodesBuilder.UNKNOWN_MAX_ID) .hasLabelInformation(hasLabelInformation) .hasProperties(!propertyColumns.isEmpty()) + .propertyStateOverride(PropertyState.TRANSIENT) .build(); nodeSubscriber.initialize(subscription.fieldNames(), this.nodesBuilder); @@ -99,12 +96,7 @@ void updateCounts(BatchLoadResult result) { @Override Nodes result() { - var nodes = nodesBuilder.build(highestNodeId); - var idMap = nodes.idMap(); - var nodeProperties = nodes.properties().propertyValues(); - var nodePropertiesWithPropertyMappings = propertiesWithPropertyMappings(nodeProperties); - - return Nodes.of(idMap, nodePropertiesWithPropertyMappings, PropertyState.TRANSIENT); + return nodesBuilder.build(highestNodeId); } @Override @@ -121,16 +113,4 @@ Set getReservedColumns() { QueryType queryType() { return QueryType.NODE; } - - private static Map propertiesWithPropertyMappings(Map properties) { - return properties.entrySet() - .stream() - .collect(Collectors.toMap( - propertiesByKey -> PropertyMapping.of( - propertiesByKey.getKey(), - propertiesByKey.getValue().valueType().fallbackValue() - ), - Map.Entry::getValue - )); - } } diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index 206d27bda9f..ababb4a834e 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -26,10 +26,12 @@ import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.ImmutableGraphSchema; import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.compat.CompatUserAggregator; @@ -668,7 +670,12 @@ private static NodeSchema nodeSchemaWithProperties( propertyMap.forEach((propertyName, nodeProperties) -> { nodeSchema.getOrCreateLabel(nodeLabel).addProperty( propertyName, - nodeProperties.valueType() + PropertySchema.of( + propertyName, + nodeProperties.valueType(), + nodeProperties.valueType().fallbackValue(), + PropertyState.TRANSIENT + ) ); }); }); From 6dce7468bf11c43e983ffa4a07b6c2b016fc4d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 18 Jan 2023 10:38:32 +0100 Subject: [PATCH 066/400] Remove `Override` suffix Co-authored-by: Martin Junghanns --- .../java/org/neo4j/gds/core/loading/CypherNodeLoader.java | 2 +- .../org/neo4j/gds/core/loading/construction/GraphFactory.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java index ae95c001f01..f6b85b0a2ca 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherNodeLoader.java @@ -67,7 +67,7 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { .maxOriginalId(NodesBuilder.UNKNOWN_MAX_ID) .hasLabelInformation(hasLabelInformation) .hasProperties(!propertyColumns.isEmpty()) - .propertyStateOverride(PropertyState.TRANSIENT) + .propertyState(PropertyState.TRANSIENT) .build(); nodeSubscriber.initialize(subscription.fieldNames(), this.nodesBuilder); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 85ea165feea..8cfcf946d07 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -90,7 +90,7 @@ static NodesBuilder nodesBuilder( Optional hasProperties, Optional deduplicateIds, Optional concurrency, - Optional propertyStateOverride + Optional propertyState ) { boolean labelInformation = nodeSchema .map(schema -> !(schema.availableLabels().isEmpty() && schema.containsOnlyAllNodesLabel())) @@ -126,7 +126,7 @@ static NodesBuilder nodesBuilder( labelInformation, hasProperties.orElse(false), deduplicate, - __ -> propertyStateOverride.orElse(PropertyState.PERSISTENT) + __ -> propertyState.orElse(PropertyState.PERSISTENT) )); } From fce4f569f127fe3df299cbc8f69bb5881e90cf45 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 18 Jan 2023 11:12:27 +0100 Subject: [PATCH 067/400] Verify that UNKNOWN_MAX_ID is handled correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Sören Reichardt --- .../neo4j/gds/core/loading/ArrayIdMap.java | 20 +++++++++---------- .../gds/core/loading/ArrayIdMapBuilder.java | 4 ++-- .../core/loading/ArrayIdMapBuilderOps.java | 14 ++++++------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java index 6f226f362ac..04787cc2158 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMap.java @@ -61,8 +61,8 @@ public class ArrayIdMap extends LabeledIdMap { private final long highestNeoId; - private final HugeLongArray graphIds; - private final HugeSparseLongArray nodeToGraphIds; + private final HugeLongArray internalToOriginalIds; + private final HugeSparseLongArray originalToInternalIds; public static MemoryEstimation memoryEstimation() { return ESTIMATION; @@ -72,26 +72,26 @@ public static MemoryEstimation memoryEstimation() { * initialize the map with pre-built sub arrays */ public ArrayIdMap( - HugeLongArray graphIds, - HugeSparseLongArray nodeToGraphIds, + HugeLongArray internalToOriginalIds, + HugeSparseLongArray originalToInternalIds, LabelInformation labelInformation, long nodeCount, long highestNeoId ) { super(labelInformation, nodeCount); - this.graphIds = graphIds; - this.nodeToGraphIds = nodeToGraphIds; + this.internalToOriginalIds = internalToOriginalIds; + this.originalToInternalIds = originalToInternalIds; this.highestNeoId = highestNeoId; } @Override public long toMappedNodeId(long originalNodeId) { - return nodeToGraphIds.get(originalNodeId); + return originalToInternalIds.get(originalNodeId); } @Override public long toOriginalNodeId(long mappedNodeId) { - return graphIds.get(mappedNodeId); + return internalToOriginalIds.get(mappedNodeId); } @Override @@ -106,7 +106,7 @@ public IdMap rootIdMap() { @Override public boolean contains(final long originalNodeId) { - return nodeToGraphIds.contains(originalNodeId); + return originalToInternalIds.contains(originalNodeId); } @Override @@ -141,7 +141,7 @@ public Optional withFilteredLabels(Collection nodeLabe HugeSparseLongArray newNodeToGraphIds = ArrayIdMapBuilderOps.buildSparseIdMap( newNodeCount, - nodeToGraphIds.capacity(), + originalToInternalIds.capacity(), concurrency, newGraphIds ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java index 28d1357a957..c31e30aa206 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilder.java @@ -69,8 +69,8 @@ public IdMap build( ) { adders.close(); long nodeCount = this.size(); - var graphIds = this.array(); - return ArrayIdMapBuilderOps.build(graphIds, nodeCount, labelInformationBuilder, highestNodeId, concurrency); + HugeLongArray internalToOriginalIds = this.array(); + return ArrayIdMapBuilderOps.build(internalToOriginalIds, nodeCount, labelInformationBuilder, highestNodeId, concurrency); } public HugeLongArray array() { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java index ba44e264b4a..cdc281241c8 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ArrayIdMapBuilderOps.java @@ -31,28 +31,28 @@ public final class ArrayIdMapBuilderOps { static ArrayIdMap build( - HugeLongArray graphIds, + HugeLongArray internalToOriginalIds, long nodeCount, LabelInformation.Builder labelInformationBuilder, long highestNodeId, int concurrency ) { if (highestNodeId == NodesBuilder.UNKNOWN_MAX_ID) { - highestNodeId = graphIds.asNodeProperties().getMaxLongPropertyValue().orElse(NodesBuilder.UNKNOWN_MAX_ID); + highestNodeId = internalToOriginalIds.asNodeProperties().getMaxLongPropertyValue().orElse(NodesBuilder.UNKNOWN_MAX_ID); } - HugeSparseLongArray nodeToGraphIds = buildSparseIdMap( + HugeSparseLongArray originalToInternalIds = buildSparseIdMap( nodeCount, highestNodeId, concurrency, - graphIds + internalToOriginalIds ); - var labelInformation = labelInformationBuilder.build(nodeCount, nodeToGraphIds::get); + var labelInformation = labelInformationBuilder.build(nodeCount, originalToInternalIds::get); return new ArrayIdMap( - graphIds, - nodeToGraphIds, + internalToOriginalIds, + originalToInternalIds, labelInformation, nodeCount, highestNodeId From 4f092ea6553d2e9c7a234df1f51d7d55990b9f6c Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 18 Jan 2023 16:20:57 +0100 Subject: [PATCH 068/400] Build node schema within NodesBuilder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Sören Reichardt --- .../org/neo4j/gds/core/loading/Nodes.java | 9 +- .../loading/construction/GraphFactory.java | 2 + .../NodeLabelTokenToPropertyKeys.java | 137 ++++++++++++++++++ .../loading/construction/NodesBuilder.java | 46 ++++-- .../loading/construction/PropertyValues.java | 13 ++ .../NodeLabelTokenToPropertyKeysTest.java | 118 +++++++++++++++ 6 files changed, 313 insertions(+), 12 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java create mode 100644 core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 33695c63970..bbbd4500c1a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -27,12 +27,15 @@ import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.schema.NodeSchema; import java.util.Map; @ValueClass public interface Nodes { + NodeSchema schema(); + IdMap idMap(); @Value.Default @@ -41,11 +44,11 @@ default NodePropertyStore properties() { } static Nodes of(IdMap idmap) { - return ImmutableNodes.of(idmap, NodePropertyStore.empty()); + return ImmutableNodes.of(NodeSchema.empty(), idmap, NodePropertyStore.empty()); } static Nodes of(IdMap idmap, NodePropertyStore nodePropertyStore) { - return ImmutableNodes.of(idmap, nodePropertyStore); + return ImmutableNodes.of(NodeSchema.empty(), idmap, nodePropertyStore); } static Nodes of(IdMap idMap, Map properties, PropertyState propertyState) { @@ -61,6 +64,6 @@ static Nodes of(IdMap idMap, Map properties : nodeProperties.valueType().fallbackValue() ) )); - return ImmutableNodes.of(idMap, builder.build()); + return ImmutableNodes.of(NodeSchema.empty(), idMap, builder.build()); } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 8cfcf946d07..967d5b96c78 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -121,6 +121,7 @@ static NodesBuilder nodesBuilder( maxOriginalId, threadCount, TokenToNodeLabelMap.lazy(), + NodeLabelTokenToPropertyKeys.lazy(), new ConcurrentHashMap<>(), idMapBuilder, labelInformation, @@ -161,6 +162,7 @@ private static NodesBuilder fromSchema( maxOriginalId, concurrency, TokenToNodeLabelMap.fixed(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping), + NodeLabelTokenToPropertyKeys.fixed(nodeSchema), new ConcurrentHashMap<>(propertyBuildersByPropertyKey), idMapBuilder, hasLabelInformation, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java new file mode 100644 index 00000000000..d5fc160dfbf --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.PropertySchema; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +abstract class NodeLabelTokenToPropertyKeys { + + /** + * Creates a thread-safe, mutable mapping. + *

+ * The property schemas are inferred from the input data. + */ + static NodeLabelTokenToPropertyKeys lazy() { + return new Lazy(); + } + + /** + * Creates thread-safe, immutable mapping. + *

+ * The property schemas are inferred from given schema. + */ + static NodeLabelTokenToPropertyKeys fixed(NodeSchema nodeSchema) { + return new Fixed(nodeSchema); + } + + /** + * Assign the given property keys to the given label token. + *

+ * If the token is already present, the property keys are added with set semantics. + */ + abstract void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys); + + /** + * Return the property schemas for the given node label. + */ + abstract Map propertySchemas( + NodeLabel nodeLabelToken, + Map importPropertySchemas + ); + + static class Fixed extends NodeLabelTokenToPropertyKeys { + + private final NodeSchema nodeSchema; + + Fixed(NodeSchema nodeSchema) { + this.nodeSchema = nodeSchema; + } + + @Override + void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys) { + // silence is golden + } + + @Override + Map propertySchemas( + NodeLabel nodeLabel, + Map importPropertySchemas + ) { + var inputPropertySchemas = nodeSchema.get(nodeLabel).properties(); + var loadPropertySchemas = importPropertySchemas + .entrySet() + .stream() + .filter(entry -> inputPropertySchemas.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + if (!inputPropertySchemas.equals(loadPropertySchemas)) { + throw new IllegalStateException( + "Property schemas inferred from loading do not match input property schema."); + } + + return inputPropertySchemas; + } + } + + static class Lazy extends NodeLabelTokenToPropertyKeys { + + private final ConcurrentHashMap> labelToPropertyKeys; + + Lazy() { + this.labelToPropertyKeys = new ConcurrentHashMap<>(); + } + + @Override + void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys) { + this.labelToPropertyKeys.compute(nodeLabelToken, (token, propertyKeySet) -> { + var keys = (propertyKeySet == null) ? new HashSet() : propertyKeySet; + propertyKeys.forEach(keys::add); + return keys; + }); + + } + + @Override + Map propertySchemas( + NodeLabel nodeLabel, + Map importPropertySchemas + ) { + return labelToPropertyKeys.keySet().stream() + .filter(nodeLabelToken -> { + for (int i = 0; i < nodeLabelToken.size(); i++) { + if (nodeLabelToken.get(i).equals(nodeLabel)) { + return true; + } + } + return false; + }) + .flatMap(nodeLabelToken -> this.labelToPropertyKeys.get(nodeLabelToken).stream()) + .collect(Collectors.toMap(propertyKey -> propertyKey, importPropertySchemas::get)); + } + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 22ee871fdc8..7f1c8a5cb79 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -29,6 +29,7 @@ import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.compat.LongPropertyReference; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -56,6 +57,7 @@ import java.util.concurrent.atomic.LongAdder; import java.util.function.Function; import java.util.function.LongPredicate; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toMap; @@ -77,12 +79,14 @@ public final class NodesBuilder { private final NodeImporter nodeImporter; private final ConcurrentMap propertyBuildersByPropertyKey; - private final boolean hasProperties; + + private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; NodesBuilder( long maxOriginalId, int concurrency, TokenToNodeLabelMap tokenToNodeLabelMap, + NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, ConcurrentMap propertyBuildersByPropertyKey, IdMapBuilder idMapBuilder, boolean hasLabelInformation, @@ -92,13 +96,13 @@ public final class NodesBuilder { ) { this.maxOriginalId = maxOriginalId; this.concurrency = concurrency; + this.nodeLabelTokenToPropertyKeys = nodeLabelTokenToPropertyKeys; this.idMapBuilder = idMapBuilder; this.propertyStates = propertyStates; this.labelInformationBuilder = !hasLabelInformation ? LabelInformationBuilders.allNodes() : LabelInformationBuilders.multiLabelWithCapacity(maxOriginalId + 1); this.propertyBuildersByPropertyKey = propertyBuildersByPropertyKey; - this.hasProperties = hasProperties; this.importedNodes = new LongAdder(); this.nodeImporter = new NodeImporter( idMapBuilder, @@ -123,6 +127,7 @@ public final class NodesBuilder { hasLabelInformation, hasProperties, tokenToNodeLabelMap, + nodeLabelTokenToPropertyKeys, propertyBuilderFn ) ); @@ -200,13 +205,28 @@ public Nodes build(long highestNeoId) { this.threadLocalBuilder.close(); var idMap = this.idMapBuilder.build(labelInformationBuilder, highestNeoId, concurrency); + var nodeProperties = buildProperties(idMap); + var nodeSchema = buildNodeSchema(idMap, nodeProperties); + var nodePropertyStore = NodePropertyStore.builder().properties(nodeProperties).build(); + + return ImmutableNodes.builder() + .schema(nodeSchema) + .idMap(idMap) + .properties(nodePropertyStore) + .build(); + } - var nodeImportResultBuilder = ImmutableNodes.builder().idMap(idMap); - if (hasProperties) { - var nodeProperties = buildProperties(idMap); - nodeImportResultBuilder.properties(NodePropertyStore.builder().properties(nodeProperties).build()); - } - return nodeImportResultBuilder.build(); + private NodeSchema buildNodeSchema(IdMap idMap, Map nodeProperties) { + var nodeSchema = NodeSchema.empty(); + var importPropertySchemas = nodeProperties + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().propertySchema())); + + idMap.availableNodeLabels().forEach(label -> { + nodeSchema.addLabel(label, this.nodeLabelTokenToPropertyKeys.propertySchemas(label, importPropertySchemas)); + }); + return nodeSchema; } private Map buildProperties(IdMap idMap) { @@ -216,7 +236,11 @@ private Map buildProperties(IdMap idMap) { )); } - private static NodeProperty entryToNodeProperty(Map.Entry entry, PropertyState propertyState, IdMap idMap) { + private static NodeProperty entryToNodeProperty( + Map.Entry entry, + PropertyState propertyState, + IdMap idMap + ) { var nodePropertyValues = entry.getValue().build(idMap); var valueType = nodePropertyValues.valueType(); return ImmutableNodeProperty.builder() @@ -264,6 +288,7 @@ private static class ThreadLocalBuilder implements AutoCloseable { private final LongAdder importedNodes; private final LongPredicate seenNodeIdPredicate; private final TokenToNodeLabelMap tokenToNodeLabelMap; + private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; private final NodesBatchBuffer buffer; private final Function propertyBuilderFn; private final NodeImporter nodeImporter; @@ -277,11 +302,13 @@ private static class ThreadLocalBuilder implements AutoCloseable { boolean hasLabelInformation, boolean hasProperties, TokenToNodeLabelMap tokenToNodeLabelMap, + NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, Function propertyBuilderFn ) { this.importedNodes = importedNodes; this.seenNodeIdPredicate = seenNodeIdPredicate; this.tokenToNodeLabelMap = tokenToNodeLabelMap; + this.nodeLabelTokenToPropertyKeys = nodeLabelTokenToPropertyKeys; this.propertyBuilderFn = propertyBuilderFn; this.buffer = new NodesBatchBufferBuilder() @@ -309,6 +336,7 @@ public void addNode(long originalId, NodeLabelToken nodeLabels) { public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues properties) { if (!seenNodeIdPredicate.test(originalId)) { long[] labels = labelTokens(nodeLabels); + this.nodeLabelTokenToPropertyKeys.add(nodeLabels, properties.propertyKeys()); int propertyReference = batchNodeProperties.size(); batchNodeProperties.add(properties); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java index d018bf817ce..660aacf2e69 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java @@ -25,6 +25,7 @@ import org.neo4j.values.virtual.MapValue; import java.util.Map; +import java.util.Set; import java.util.function.BiConsumer; public abstract class PropertyValues { @@ -33,6 +34,8 @@ public abstract class PropertyValues { public abstract boolean isEmpty(); + public abstract Iterable propertyKeys(); + public static PropertyValues of(MapValue mapValue) { return new CypherPropertyValues(mapValue); } @@ -57,6 +60,11 @@ public void forEach(BiConsumer consumer) { public boolean isEmpty() { return this.properties.isEmpty(); } + + @Override + public Set propertyKeys() { + return this.properties.keySet(); + } } private static final class CypherPropertyValues extends PropertyValues { @@ -79,5 +87,10 @@ public void forEach(BiConsumer consumer) { public boolean isEmpty() { return this.properties.isEmpty(); } + + @Override + public Iterable propertyKeys() { + return this.properties.keySet(); + } } } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java new file mode 100644 index 00000000000..30381c93cba --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.nodeproperties.ValueType; +import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.PropertySchema; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class NodeLabelTokenToPropertyKeysTest { + + @Test + void testPropertySchemasLazy() { + var mapping = NodeLabelTokenToPropertyKeys.lazy(); + + mapping.add(NodeLabelTokens.ofStrings("A"), List.of("foo", "bar")); + mapping.add(NodeLabelTokens.ofStrings("A"), List.of("baz")); + mapping.add(NodeLabelTokens.ofStrings("B"), List.of("baz", "bob")); + mapping.add(NodeLabelTokens.ofStrings("C"), List.of()); + + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bob", PropertySchema.of("bob", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + ); + + assertThat(mapping.propertySchemas(NodeLabel.of("A"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("B"), importPropertySchemas)).isEqualTo(Map.of( + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bob", PropertySchema.of("bob", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("C"), importPropertySchemas)).isEqualTo(Map.of()); + } + + @Test + void testPropertySchemasFixed() { + var nodeSchema = NodeSchema.empty() + .addLabel(NodeLabel.of("A"), Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG), + "bar", PropertySchema.of("bar", ValueType.DOUBLE), + "baz", PropertySchema.of("baz", ValueType.LONG) + )) + .addLabel(NodeLabel.of("B"), Map.of( + "baz", PropertySchema.of("baz", ValueType.LONG), + "bob", PropertySchema.of("bob", ValueType.LONG) + )) + .addLabel(NodeLabel.of("C")); + + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bob", PropertySchema.of("bob", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + ); + + var mapping = NodeLabelTokenToPropertyKeys.fixed(nodeSchema); + + assertThat(mapping.propertySchemas(NodeLabel.of("A"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("B"), importPropertySchemas)).isEqualTo(Map.of( + "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bob", PropertySchema.of("bob", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("C"), importPropertySchemas)).isEqualTo(Map.of()); + } + + @Test + void shouldFailOnInconsistentPropertySchemas() { + var nodeSchema = NodeSchema.empty() + .addLabel(NodeLabel.of("A"), Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG) + )); + + var fixed = NodeLabelTokenToPropertyKeys.fixed(nodeSchema); + + assertThatThrownBy(() -> fixed.propertySchemas( + NodeLabel.of("A"), + Map.of("bar", PropertySchema.of("bar", ValueType.DOUBLE)) + )) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Property schemas inferred from loading do not match input property schema."); + } +} + From 36268c00d6b3e9b490d51dfd654bc3c4db12ebaa Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 18 Jan 2023 16:24:34 +0100 Subject: [PATCH 069/400] Minor name and visibility refactoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Sören Reichardt --- .../loading/construction/GraphFactory.java | 4 +-- .../NodeLabelTokenToPropertyKeys.java | 4 +-- .../loading/construction/NodesBuilder.java | 22 ++++++++-------- ...odeLabelMap.java => TokenToNodeLabel.java} | 26 +++++++++---------- 4 files changed, 28 insertions(+), 28 deletions(-) rename core/src/main/java/org/neo4j/gds/core/loading/construction/{TokenToNodeLabelMap.java => TokenToNodeLabel.java} (82%) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 967d5b96c78..b0da169f8b7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -120,7 +120,7 @@ static NodesBuilder nodesBuilder( )).orElseGet(() -> new NodesBuilder( maxOriginalId, threadCount, - TokenToNodeLabelMap.lazy(), + TokenToNodeLabel.lazy(), NodeLabelTokenToPropertyKeys.lazy(), new ConcurrentHashMap<>(), idMapBuilder, @@ -161,7 +161,7 @@ private static NodesBuilder fromSchema( return new NodesBuilder( maxOriginalId, concurrency, - TokenToNodeLabelMap.fixed(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping), + TokenToNodeLabel.fixed(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping), NodeLabelTokenToPropertyKeys.fixed(nodeSchema), new ConcurrentHashMap<>(propertyBuildersByPropertyKey), idMapBuilder, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java index d5fc160dfbf..1a7d56c1918 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -64,7 +64,7 @@ abstract Map propertySchemas( Map importPropertySchemas ); - static class Fixed extends NodeLabelTokenToPropertyKeys { + private static class Fixed extends NodeLabelTokenToPropertyKeys { private final NodeSchema nodeSchema; @@ -98,7 +98,7 @@ Map propertySchemas( } } - static class Lazy extends NodeLabelTokenToPropertyKeys { + private static class Lazy extends NodeLabelTokenToPropertyKeys { private final ConcurrentHashMap> labelToPropertyKeys; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 7f1c8a5cb79..372a3c10063 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -85,7 +85,7 @@ public final class NodesBuilder { NodesBuilder( long maxOriginalId, int concurrency, - TokenToNodeLabelMap tokenToNodeLabelMap, + TokenToNodeLabel tokenToNodeLabel, NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, ConcurrentMap propertyBuildersByPropertyKey, IdMapBuilder idMapBuilder, @@ -107,7 +107,7 @@ public final class NodesBuilder { this.nodeImporter = new NodeImporter( idMapBuilder, labelInformationBuilder, - tokenToNodeLabelMap.labelTokenNodeLabelMapping(), + tokenToNodeLabel.labelTokenNodeLabelMapping(), hasProperties ); @@ -126,7 +126,7 @@ public final class NodesBuilder { seenNodeIdPredicate, hasLabelInformation, hasProperties, - tokenToNodeLabelMap, + tokenToNodeLabel, nodeLabelTokenToPropertyKeys, propertyBuilderFn ) @@ -287,7 +287,7 @@ private static class ThreadLocalBuilder implements AutoCloseable { private final LongAdder importedNodes; private final LongPredicate seenNodeIdPredicate; - private final TokenToNodeLabelMap tokenToNodeLabelMap; + private final TokenToNodeLabel tokenToNodeLabel; private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; private final NodesBatchBuffer buffer; private final Function propertyBuilderFn; @@ -301,13 +301,13 @@ private static class ThreadLocalBuilder implements AutoCloseable { LongPredicate seenNodeIdPredicate, boolean hasLabelInformation, boolean hasProperties, - TokenToNodeLabelMap tokenToNodeLabelMap, + TokenToNodeLabel tokenToNodeLabel, NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, Function propertyBuilderFn ) { this.importedNodes = importedNodes; this.seenNodeIdPredicate = seenNodeIdPredicate; - this.tokenToNodeLabelMap = tokenToNodeLabelMap; + this.tokenToNodeLabel = tokenToNodeLabel; this.nodeLabelTokenToPropertyKeys = nodeLabelTokenToPropertyKeys; this.propertyBuilderFn = propertyBuilderFn; @@ -323,7 +323,7 @@ private static class ThreadLocalBuilder implements AutoCloseable { public void addNode(long originalId, NodeLabelToken nodeLabels) { if (!seenNodeIdPredicate.test(originalId)) { - long[] labels = labelTokens(nodeLabels); + long[] labels = getOrCreateLabelTokens(nodeLabels); buffer.add(originalId, LongPropertyReference.empty(), labels); if (buffer.isFull()) { @@ -335,7 +335,7 @@ public void addNode(long originalId, NodeLabelToken nodeLabels) { public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues properties) { if (!seenNodeIdPredicate.test(originalId)) { - long[] labels = labelTokens(nodeLabels); + long[] labels = getOrCreateLabelTokens(nodeLabels); this.nodeLabelTokenToPropertyKeys.add(nodeLabels, properties.propertyKeys()); int propertyReference = batchNodeProperties.size(); @@ -349,14 +349,14 @@ public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues p } } - private long[] labelTokens(NodeLabelToken nodeLabels) { + private long[] getOrCreateLabelTokens(NodeLabelToken nodeLabels) { if (nodeLabels.isEmpty()) { return anyLabelArray(); } long[] labelIds = new long[nodeLabels.size()]; for (int i = 0; i < labelIds.length; i++) { - labelIds[i] = tokenToNodeLabelMap.getTokenForNodeLabel(nodeLabels.get(i)); + labelIds[i] = tokenToNodeLabel.getOrCreateToken(nodeLabels.get(i)); } return labelIds; @@ -412,7 +412,7 @@ private int importProperty(long neoNodeId, String propertyKey, Value value) { private long[] anyLabelArray() { var anyLabelArray = this.anyLabelArray; if (anyLabelArray[0] == NOT_INITIALIZED) { - anyLabelArray[0] = tokenToNodeLabelMap.getTokenForNodeLabel(NodeLabel.ALL_NODES); + anyLabelArray[0] = tokenToNodeLabel.getOrCreateToken(NodeLabel.ALL_NODES); } return anyLabelArray; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabel.java similarity index 82% rename from core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java rename to core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabel.java index 672f58aa7a6..59bc6b8dc0d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabelMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabel.java @@ -32,23 +32,23 @@ import static org.neo4j.gds.core.GraphDimensions.NO_SUCH_LABEL; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -public abstract class TokenToNodeLabelMap { +abstract class TokenToNodeLabel { final ObjectIntMap elementIdentifierLabelTokenMapping; final IntObjectHashMap> labelTokenNodeLabelMapping; - public static TokenToNodeLabelMap fixed( + static TokenToNodeLabel fixed( ObjectIntMap elementIdentifierLabelTokenMapping, IntObjectHashMap> labelTokenNodeLabelMapping ) { - return new FixedTokenToNodeLabelMap(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); + return new FixedTokenToNodeLabel(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); } - public static TokenToNodeLabelMap lazy() { - return new LazyTokenToNodeLabelMap(); + static TokenToNodeLabel lazy() { + return new LazyTokenToNodeLabel(); } - TokenToNodeLabelMap( + TokenToNodeLabel( ObjectIntMap elementIdentifierLabelTokenMapping, IntObjectHashMap> labelTokenNodeLabelMapping ) { @@ -60,11 +60,11 @@ IntObjectHashMap> labelTokenNodeLabelMapping() { return this.labelTokenNodeLabelMapping; } - public abstract int getTokenForNodeLabel(NodeLabel nodeLabel); + abstract int getOrCreateToken(NodeLabel nodeLabel); - static class FixedTokenToNodeLabelMap extends TokenToNodeLabelMap { + private static class FixedTokenToNodeLabel extends TokenToNodeLabel { - FixedTokenToNodeLabelMap( + FixedTokenToNodeLabel( ObjectIntMap elementIdentifierLabelTokenMapping, IntObjectHashMap> labelTokenNodeLabelMapping ) { @@ -72,7 +72,7 @@ static class FixedTokenToNodeLabelMap extends TokenToNodeLabelMap { } @Override - public int getTokenForNodeLabel(NodeLabel nodeLabel) { + public int getOrCreateToken(NodeLabel nodeLabel) { if (!elementIdentifierLabelTokenMapping.containsKey(nodeLabel)) { throw new IllegalArgumentException(formatWithLocale("No token was specified for node label %s", nodeLabel)); } @@ -80,19 +80,19 @@ public int getTokenForNodeLabel(NodeLabel nodeLabel) { } } - static class LazyTokenToNodeLabelMap extends TokenToNodeLabelMap { + private static class LazyTokenToNodeLabel extends TokenToNodeLabel { private final Lock lock; private int nextLabelId; - LazyTokenToNodeLabelMap() { + LazyTokenToNodeLabel() { super(new ObjectIntScatterMap<>(), new IntObjectHashMap<>()); this.lock = new ReentrantLock(true); this.nextLabelId = 0; } @Override - public int getTokenForNodeLabel(NodeLabel nodeLabel) { + public int getOrCreateToken(NodeLabel nodeLabel) { var token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); if (token == NO_SUCH_LABEL) { lock.lock(); From 597ae778b89a34cda08d166a62bebd63ef3905a2 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 19 Jan 2023 08:39:57 +0100 Subject: [PATCH 070/400] Make Nodes#nodeSchema default (for now) --- core/src/main/java/org/neo4j/gds/core/loading/Nodes.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index bbbd4500c1a..a94bd8a227d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -34,7 +34,10 @@ @ValueClass public interface Nodes { - NodeSchema schema(); + @Value.Default + default NodeSchema schema() { + return NodeSchema.empty(); + } IdMap idMap(); From d9f4dd49b1f421ace2bd025fbe0f879e702d2015 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 19 Jan 2023 10:39:14 +0100 Subject: [PATCH 071/400] Fix key and type validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Sören Reichardt --- .../NodeLabelTokenToPropertyKeys.java | 45 +++++++++++++++---- .../NodeLabelTokenToPropertyKeysTest.java | 32 +++++++++++-- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java index 1a7d56c1918..78a6f238d49 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -22,6 +22,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.api.schema.PropertySchema; +import org.neo4j.gds.utils.StringJoining; import java.util.HashSet; import java.util.Map; @@ -82,19 +83,45 @@ Map propertySchemas( NodeLabel nodeLabel, Map importPropertySchemas ) { - var inputPropertySchemas = nodeSchema.get(nodeLabel).properties(); - var loadPropertySchemas = importPropertySchemas - .entrySet() + var userDefinedPropertySchemas = nodeSchema.get(nodeLabel).properties(); + + // We validate that the property schemas we read during import have + // at least a matching key and a matching type. We cannot do an + // equality check because we cannot infer the default value or the + // property state from just looking at the values. + var overlap = importPropertySchemas + .keySet() .stream() - .filter(entry -> inputPropertySchemas.containsKey(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .filter(userDefinedPropertySchemas::containsKey) + .collect(Collectors.toSet()); + + if (overlap.size() < userDefinedPropertySchemas.size()) { + var keySet = new HashSet<>(userDefinedPropertySchemas.keySet()); + keySet.removeAll(overlap); + throw new IllegalStateException("Missing node properties during import. " + + "The following keys were part of the schema, " + + "but not contained in the input data: " + + StringJoining.join(keySet) + ); + } - if (!inputPropertySchemas.equals(loadPropertySchemas)) { - throw new IllegalStateException( - "Property schemas inferred from loading do not match input property schema."); + // We got the same keys and can check the types. + var keysWithIncompatibleTypes = overlap.stream() + .filter(propertyKey -> userDefinedPropertySchemas + .get(propertyKey) + .valueType() != importPropertySchemas + .get(propertyKey) + .valueType()).collect(Collectors.toSet()); + + if (!keysWithIncompatibleTypes.isEmpty()) { + throw new IllegalStateException("Incompatible value types between input schema and input data. " + + "The following keys have incompatible types: " + + StringJoining.join(keysWithIncompatibleTypes) + ); } - return inputPropertySchemas; + + return userDefinedPropertySchemas; } } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java index 30381c93cba..a41e544706c 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -99,20 +99,46 @@ void testPropertySchemasFixed() { } @Test - void shouldFailOnInconsistentPropertySchemas() { + void shouldFailForMissingProperties() { var nodeSchema = NodeSchema.empty() .addLabel(NodeLabel.of("A"), Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG), + "baz", PropertySchema.of("baz", ValueType.LONG) + )); + + var fixed = NodeLabelTokenToPropertyKeys.fixed(nodeSchema); + + assertThatThrownBy(() -> fixed.propertySchemas( + NodeLabel.of("A"), + Map.of( "foo", PropertySchema.of("foo", ValueType.LONG) + ) + )) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Missing node properties during import.") + .hasMessageContaining("['baz']"); + } + + @Test + void shouldFailForIncompatibleTypes() { + var nodeSchema = NodeSchema.empty() + .addLabel(NodeLabel.of("A"), Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG), + "baz", PropertySchema.of("baz", ValueType.LONG) )); var fixed = NodeLabelTokenToPropertyKeys.fixed(nodeSchema); assertThatThrownBy(() -> fixed.propertySchemas( NodeLabel.of("A"), - Map.of("bar", PropertySchema.of("bar", ValueType.DOUBLE)) + Map.of( + "foo", PropertySchema.of("foo", ValueType.DOUBLE), + "baz", PropertySchema.of("baz", ValueType.LONG) + ) )) .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Property schemas inferred from loading do not match input property schema."); + .hasMessageContaining("Incompatible value types between input schema and input data.") + .hasMessageContaining("['foo']"); } } From fe8eaaea74cf569a86ac2f2384cedd3d92ac9fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 19 Jan 2023 11:26:22 +0100 Subject: [PATCH 072/400] Use Nodes#nodeSchema in arrow import Co-authored-by: Martin Junghanns --- .../gds/core/loading/CSRGraphStoreUtil.java | 19 ++----------------- .../neo4j/gds/core/loading/CypherFactory.java | 1 - .../gds/core/loading/LazyIdMapBuilder.java | 4 ++++ .../neo4j/gds/core/loading/NativeFactory.java | 1 - .../org/neo4j/gds/core/loading/Nodes.java | 18 ++++++++++++++++++ 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index 7915d27398c..2e4cf64f48c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.loading; import org.jetbrains.annotations.NotNull; -import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.Graph; @@ -43,7 +42,6 @@ import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.values.storable.NumberType; -import java.util.Collection; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -203,25 +201,12 @@ public static void extractNodeProperties( nodeImportResultBuilder.properties(propertyStoreBuilder.build()); } + // TODO: remove this method public static GraphSchema computeGraphSchema( Nodes nodes, - Function> propertiesByLabel, RelationshipImportResult relationshipImportResult ) { - var nodeProperties = nodes.properties().properties(); - - var nodeSchema = NodeSchema.empty(); - for (var label : nodes.idMap().availableNodeLabels()) { - var entry = nodeSchema.getOrCreateLabel(label); - for (var propertyKey : propertiesByLabel.apply(label)) { - entry.addProperty( - propertyKey, - nodeProperties.get(propertyKey).propertySchema() - ); - } - } - nodes.idMap().availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); - + var nodeSchema = nodes.schema(); var relationshipSchema = RelationshipSchema.empty(); relationshipImportResult.importResults().forEach(((relationshipType, singleTypeRelationshipImportResult) -> { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index 700481b2dcf..731c5e34917 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -124,7 +124,6 @@ protected GraphSchema computeGraphSchema( ) { return CSRGraphStoreUtil.computeGraphSchema( nodes, - (__) -> nodes.properties().keySet(), relationshipImportResult ); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index e05fea8ba7a..937a9c712ce 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -23,6 +23,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodeLabelToken; import org.neo4j.gds.core.loading.construction.NodesBuilder; @@ -110,6 +111,8 @@ public interface HighLimitIdMapAndProperties { PartialIdMap intermediateIdMap(); + NodeSchema schema(); + Optional> nodeProperties(); } @@ -141,6 +144,7 @@ public OptionalLong rootNodeCount() { .builder() .idMap(idMap) .intermediateIdMap(partialIdMap) + .schema(nodes.schema()) .nodeProperties(nodes.properties().propertyValues()) .build(); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java index bba0021e742..89df81b651d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java @@ -301,7 +301,6 @@ protected GraphSchema computeGraphSchema( ) { return CSRGraphStoreUtil.computeGraphSchema( nodes, - (label) -> storeConfig.nodeProjections().projections().get(label).properties().propertyKeys(), relationshipImportResult ); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index a94bd8a227d..66f131d143a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -24,6 +24,7 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; @@ -54,6 +55,23 @@ static Nodes of(IdMap idmap, NodePropertyStore nodePropertyStore) { return ImmutableNodes.of(NodeSchema.empty(), idmap, nodePropertyStore); } + static Nodes of(IdMap idMap, NodeSchema nodeSchema, Map properties, PropertyState propertyState) { + var nodePropertyStoreBuilder = NodePropertyStore.builder(); + + nodeSchema.availableLabels().forEach(nodeLabel -> { + var propertiesForLabel = nodeSchema.get(nodeLabel).properties(); + propertiesForLabel.forEach((propertyKey, propertySchema) -> nodePropertyStoreBuilder.putProperty(propertyKey, + ImmutableNodeProperty + .builder() + .propertySchema(propertySchema) + .values(properties.get(propertyKey)) + .build() + )); + }); + + return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); + } + static Nodes of(IdMap idMap, Map properties, PropertyState propertyState) { NodePropertyStore.Builder builder = NodePropertyStore.builder(); properties.forEach((mapping, nodeProperties) -> builder.putProperty( From a2e273a301a56aa1d4dd5f08572ab25ea018013a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 19 Jan 2023 12:02:20 +0100 Subject: [PATCH 073/400] Use Nodes#nodeSchema in native factory Co-authored-by: Martin Junghanns --- .../core/loading/IndexPropertyMappings.java | 15 +++++++++ .../org/neo4j/gds/core/loading/Nodes.java | 33 +++++++++++++++++++ .../core/loading/ScanningNodesImporter.java | 10 +++++- 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java b/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java index 806bc3fe778..537ea6fbe11 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java @@ -46,6 +46,7 @@ import static java.util.stream.Collectors.toMap; import static org.neo4j.gds.core.GraphDimensions.ANY_LABEL; +// TODO: should be named LoadablePropertyMappings final class IndexPropertyMappings { static LoadablePropertyMappings prepareProperties( @@ -66,6 +67,20 @@ static LoadablePropertyMappings prepareProperties( return prepareLoadableProperties(graphDimensions, transaction, storeLoadedProperties); } + static Map propertyMappingsByLabel(GraphProjectFromStoreConfig graphProjectConfig) { + return graphProjectConfig + .nodeProjections() + .projections() + .entrySet() + .stream() + .collect(toMap( + Map.Entry::getKey, + entry -> entry + .getValue() + .properties() + )); + } + private static LoadablePropertyMappings prepareLoadableProperties( GraphDimensions dimensions, TransactionContext transaction, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 66f131d143a..7d8ef2ee151 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -20,7 +20,9 @@ package org.neo4j.gds.core.loading; import org.immutables.value.Value; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.PropertyMapping; +import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; @@ -28,6 +30,7 @@ import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.schema.ImmutablePropertySchema; import org.neo4j.gds.api.schema.NodeSchema; import java.util.Map; @@ -72,6 +75,36 @@ static Nodes of(IdMap idMap, NodeSchema nodeSchema, Map propertyMappingsByLabel, + Map properties, + PropertyState propertyState + ) { + var nodeSchema = NodeSchema.empty(); + var nodePropertyStoreBuilder = NodePropertyStore.builder(); + propertyMappingsByLabel.forEach(((nodeLabel, propertyMappings) -> { + if (propertyMappings.mappings().isEmpty()) { + nodeSchema.addLabel(nodeLabel); + } else { + propertyMappings.mappings().forEach(propertyMapping -> { + var nodePropertyValues = properties.get(propertyMapping); + var propertySchema = ImmutablePropertySchema.builder() + .key(propertyMapping.propertyKey()) + .valueType(nodePropertyValues.valueType()) + .defaultValue(propertyMapping.defaultValue()) + .state(propertyState) + .build(); + + nodeSchema.addProperty(nodeLabel, propertySchema.key(), propertySchema); + nodePropertyStoreBuilder.putProperty(propertySchema.key(), ImmutableNodeProperty.of(nodePropertyValues, propertySchema)); + }); + } + })); + + return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); + } + static Nodes of(IdMap idMap, Map properties, PropertyState propertyState) { NodePropertyStore.Builder builder = NodePropertyStore.builder(); properties.forEach((mapping, nodeProperties) -> builder.putProperty( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 6f8668c8298..2cf3ae84170 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -21,7 +21,9 @@ import org.immutables.builder.Builder; import org.jetbrains.annotations.Nullable; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.PropertyMapping; +import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; @@ -50,6 +52,7 @@ public final class ScanningNodesImporter extends ScanningRecordsImporter { private final IndexPropertyMappings.LoadablePropertyMappings propertyMappings; + private final Map propertyMappingsByLabel; private final TerminationFlag terminationFlag; private final IdMapBuilder idMapBuilder; private final LabelInformation.Builder labelInformationBuilder; @@ -93,6 +96,8 @@ public static ScanningNodesImporter scanningNodesImporter( loadingContext.transactionContext() ); + var propertyMappingsByLabel = IndexPropertyMappings.propertyMappingsByLabel(graphProjectConfig); + var nodePropertyImporter = initializeNodePropertyImporter( propertyMappings, dimensions, @@ -106,6 +111,7 @@ public static ScanningNodesImporter scanningNodesImporter( progressTracker, concurrency, propertyMappings, + propertyMappingsByLabel, nodePropertyImporter, idMapBuilder, labelInformationBuilder @@ -119,6 +125,7 @@ private ScanningNodesImporter( ProgressTracker progressTracker, int concurrency, IndexPropertyMappings.LoadablePropertyMappings propertyMappings, + Map propertyMappingsByLabel, @Nullable NativeNodePropertyImporter nodePropertyImporter, IdMapBuilder idMapBuilder, LabelInformation.Builder labelInformationBuilder @@ -133,6 +140,7 @@ private ScanningNodesImporter( this.terminationFlag = loadingContext.terminationFlag(); this.propertyMappings = propertyMappings; + this.propertyMappingsByLabel = propertyMappingsByLabel; this.nodePropertyImporter = nodePropertyImporter; this.idMapBuilder = idMapBuilder; this.labelInformationBuilder = labelInformationBuilder; @@ -191,7 +199,7 @@ public Nodes build() { importPropertiesFromIndex(idMap, nodeProperties); } - return Nodes.of(idMap, nodeProperties, PropertyState.PERSISTENT); + return Nodes.of(idMap, propertyMappingsByLabel, nodeProperties, PropertyState.PERSISTENT); } private void importPropertiesFromIndex( From c6cffbc0509cbff8fba1f9f3f5ab45e127cd3282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 19 Jan 2023 12:33:49 +0100 Subject: [PATCH 074/400] Use Nodes#nodeSchema in gdl factory Co-authored-by: Martin Junghanns --- .../neo4j/gds/api/CSRGraphStoreFactory.java | 2 +- .../neo4j/gds/core/loading/CypherFactory.java | 6 +- .../org/neo4j/gds/core/loading/Nodes.java | 33 ++++---- .../NodeLabelTokenToPropertyKeys.java | 6 +- .../java/org/neo4j/gds/gdl/GdlFactory.java | 79 +++++++++++-------- 5 files changed, 70 insertions(+), 56 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java index 11fc8d4d9f2..eb5eb3a36ce 100644 --- a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java @@ -50,7 +50,7 @@ protected CSRGraphStore createGraphStore( .databaseId(DatabaseId.of(loadingContext.graphDatabaseService())) .capabilities(capabilities) .schema(computeGraphSchema(nodes, relationshipImportResult)) - .nodes(Nodes.of(nodes.idMap(), nodes.properties())) + .nodes(nodes) .relationshipImportResult(relationshipImportResult) .concurrency(graphProjectConfig.readConcurrency()) .build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index 731c5e34917..554775323cd 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -140,7 +140,7 @@ public CSRGraphStore build() { ).load(ktx.internalTransaction()); progressTracker.beginSubTask("Loading"); - var idMapAndProperties = new CypherNodeLoader( + var nodes = new CypherNodeLoader( nodeQuery(), nodeCount.rows(), cypherConfig, @@ -150,14 +150,14 @@ public CSRGraphStore build() { var relationshipsAndProperties = new CypherRelationshipLoader( relationshipQuery(), - idMapAndProperties.idMap(), + nodes.idMap(), cypherConfig, loadingContext, progressTracker ).load(ktx.internalTransaction()); var graphStore = createGraphStore( - idMapAndProperties, + nodes, relationshipsAndProperties ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 7d8ef2ee151..40953c0bd4b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -58,23 +58,6 @@ static Nodes of(IdMap idmap, NodePropertyStore nodePropertyStore) { return ImmutableNodes.of(NodeSchema.empty(), idmap, nodePropertyStore); } - static Nodes of(IdMap idMap, NodeSchema nodeSchema, Map properties, PropertyState propertyState) { - var nodePropertyStoreBuilder = NodePropertyStore.builder(); - - nodeSchema.availableLabels().forEach(nodeLabel -> { - var propertiesForLabel = nodeSchema.get(nodeLabel).properties(); - propertiesForLabel.forEach((propertyKey, propertySchema) -> nodePropertyStoreBuilder.putProperty(propertyKey, - ImmutableNodeProperty - .builder() - .propertySchema(propertySchema) - .values(properties.get(propertyKey)) - .build() - )); - }); - - return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); - } - static Nodes of( IdMap idMap, Map propertyMappingsByLabel, @@ -120,4 +103,20 @@ static Nodes of(IdMap idMap, Map properties )); return ImmutableNodes.of(NodeSchema.empty(), idMap, builder.build()); } + + static Nodes of(NodeSchema nodeSchema, IdMap idMap, Map properties, PropertyState propertyState) { + NodePropertyStore.Builder builder = NodePropertyStore.builder(); + properties.forEach((mapping, nodeProperties) -> builder.putProperty( + mapping.propertyKey(), + NodeProperty.of( + mapping.propertyKey(), + propertyState, + nodeProperties, + mapping.defaultValue().isUserDefined() + ? mapping.defaultValue() + : nodeProperties.valueType().fallbackValue() + ) + )); + return ImmutableNodes.of(nodeSchema, idMap, builder.build()); + } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java index 78a6f238d49..f57af989da7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -158,7 +158,11 @@ Map propertySchemas( return false; }) .flatMap(nodeLabelToken -> this.labelToPropertyKeys.get(nodeLabelToken).stream()) - .collect(Collectors.toMap(propertyKey -> propertyKey, importPropertySchemas::get)); + .collect(Collectors.toMap( + propertyKey -> propertyKey, + importPropertySchemas::get, + (lhs, rhs) -> lhs + )); } } } diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index 2b2688b6f77..5ea833439fa 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -34,8 +34,6 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; @@ -49,12 +47,14 @@ import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.ImmutablePropertyConfig; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; +import org.neo4j.gds.core.loading.construction.PropertyValues; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; import org.neo4j.gds.core.loading.nodeproperties.NodePropertiesFromStoreBuilder; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlSupportPerMethodExtension; +import org.neo4j.values.storable.Value; import org.neo4j.values.storable.Values; import org.s1ck.gdl.GDLHandler; import org.s1ck.gdl.model.Element; @@ -165,33 +165,33 @@ protected ProgressTracker initProgressTracker() { @Override protected GraphSchema computeGraphSchema(Nodes nodes, RelationshipImportResult relationshipImportResult) { - var nodeProperties = nodes.properties(); - var nodeSchema = NodeSchema.empty(); - gdlHandler - .getVertices() - .forEach(vertex -> { - var labels = vertex.getLabels().stream().map(NodeLabel::of).collect(Collectors.toList()); - if (labels.isEmpty()) { - labels = List.of(NodeLabel.ALL_NODES); - } - - labels.forEach(label -> vertex - .getProperties() - .forEach((propertyKey, propertyValue) -> nodeSchema - .getOrCreateLabel(label) - .addProperty( - propertyKey, - PropertySchema.of( - propertyKey, - nodeProperties.get(propertyKey).valueType(), - nodeProperties.get(propertyKey).defaultValue(), - nodeProperties.get(propertyKey).propertyState() - ) - ) - )); - }); - // in case there were no properties add all labels - nodes.idMap().availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); +// var nodeProperties = nodes.properties(); +// var nodeSchema = NodeSchema.empty(); +// gdlHandler +// .getVertices() +// .forEach(vertex -> { +// var labels = vertex.getLabels().stream().map(NodeLabel::of).collect(Collectors.toList()); +// if (labels.isEmpty()) { +// labels = List.of(NodeLabel.ALL_NODES); +// } +// +// labels.forEach(label -> vertex +// .getProperties() +// .forEach((propertyKey, propertyValue) -> nodeSchema +// .getOrCreateLabel(label) +// .addProperty( +// propertyKey, +// PropertySchema.of( +// propertyKey, +// nodeProperties.get(propertyKey).valueType(), +// nodeProperties.get(propertyKey).defaultValue(), +// nodeProperties.get(propertyKey).propertyState() +// ) +// ) +// )); +// }); +// // in case there were no properties add all labels +// nodes.idMap().availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); var relationshipSchema = relationshipImportResult.importResults().entrySet().stream().reduce( RelationshipSchema.empty(), @@ -204,7 +204,7 @@ protected GraphSchema computeGraphSchema(Nodes nodes, RelationshipImportResult r ); return GraphSchema.of( - nodeSchema, + nodes.schema(), relationshipSchema, Map.of() ); @@ -230,7 +230,9 @@ private Nodes loadNodes() { var nodesBuilder = GraphFactory.initNodesBuilder() .maxOriginalId(dimensions.highestPossibleNodeCount() - 1) .hasLabelInformation(true) + .hasProperties(true) .concurrency(1) + .propertyState(graphProjectConfig.propertyState()) .build(); gdlHandler.getVertices().forEach(vertex -> { @@ -241,15 +243,24 @@ private Nodes loadNodes() { .filter(label -> !NodeLabel.ALL_NODES.name().equals(label)) .collect(Collectors.toList()); } + + Map propertyValues = new HashMap<>(); + vertex.getProperties().forEach((propertyKey, propertyValue) -> { + if (propertyValue instanceof List) { + propertyValue = convertListProperty((List) propertyValue); + } + propertyValues.put(propertyKey, Values.of(propertyValue)); + }); + nodesBuilder.addNode( vertex.getId(), - NodeLabelTokens.of(labels) + NodeLabelTokens.of(labels), + PropertyValues.of(propertyValues) ); }); - var idMap = nodesBuilder.build().idMap(); - - return Nodes.of(idMap, loadNodeProperties(idMap), graphProjectConfig.propertyState()); + var nodes = nodesBuilder.build(); + return Nodes.of(nodes.schema(), nodes.idMap(), loadNodeProperties(nodes.idMap()), graphProjectConfig.propertyState()); } private Map loadNodeProperties(IdMap idMap) { From 71ed216170ce9cf8999388a17883e5927b9d40db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 19 Jan 2023 12:39:52 +0100 Subject: [PATCH 075/400] Avoid creation of Nodes without node schema Co-authored-by: Martin Junghanns --- .../java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java | 2 +- core/src/main/java/org/neo4j/gds/core/loading/Nodes.java | 6 ++---- .../org/neo4j/gds/graphsampling/GraphSampleConstructor.java | 4 ++-- .../java/org/neo4j/gds/beta/filter/GraphStoreFilter.java | 4 ++-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index 2e4cf64f48c..aa1cd8c8b9f 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -104,7 +104,7 @@ public static CSRGraphStore createFromGraph( // TODO: is it correct that we only use this for generated graphs? .capabilities(ImmutableStaticCapabilities.of(false)) .schema(schema) - .nodes(Nodes.of(graph.idMap(), nodeProperties)) + .nodes(ImmutableNodes.of(schema.nodeSchema(), graph.idMap(), nodeProperties)) .relationshipImportResult(relationshipImportResult) .graphProperties(GraphPropertyStore.empty()) .concurrency(concurrency) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 40953c0bd4b..c0b8748dfcc 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -20,6 +20,7 @@ package org.neo4j.gds.core.loading; import org.immutables.value.Value; +import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; @@ -50,14 +51,11 @@ default NodePropertyStore properties() { return NodePropertyStore.empty(); } + @TestOnly static Nodes of(IdMap idmap) { return ImmutableNodes.of(NodeSchema.empty(), idmap, NodePropertyStore.empty()); } - static Nodes of(IdMap idmap, NodePropertyStore nodePropertyStore) { - return ImmutableNodes.of(NodeSchema.empty(), idmap, nodePropertyStore); - } - static Nodes of( IdMap idMap, Map propertyMappingsByLabel, diff --git a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java index df488d04e11..00ed2f9cc66 100644 --- a/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java +++ b/graph-sampling/src/main/java/org/neo4j/gds/graphsampling/GraphSampleConstructor.java @@ -33,7 +33,7 @@ import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.loading.GraphStoreBuilder; -import org.neo4j.gds.core.loading.Nodes; +import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; @@ -124,7 +124,7 @@ public double evaluate(EvaluationContext context) { .databaseId(inputGraphStore.databaseId()) .capabilities(inputGraphStore.capabilities()) .schema(filteredSchema) - .nodes(Nodes.of(idMap, nodePropertyStore)) + .nodes(ImmutableNodes.of(filteredSchema.nodeSchema(), idMap, nodePropertyStore)) .relationshipImportResult(relationshipImportResult) .concurrency(config.concurrency()) .build(); diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java index 82be89e2310..be3b39d2d29 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java @@ -34,7 +34,7 @@ import org.neo4j.gds.beta.filter.expression.ValidationContext; import org.neo4j.gds.config.GraphProjectFromGraphConfig; import org.neo4j.gds.core.loading.GraphStoreBuilder; -import org.neo4j.gds.core.loading.Nodes; +import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; @@ -121,7 +121,7 @@ public static GraphStore filter( .databaseId(graphStore.databaseId()) .capabilities(graphStore.capabilities()) .schema(filteredSchema) - .nodes(Nodes.of(filteredNodes.idMap(), filteredNodes.propertyStores())) + .nodes(ImmutableNodes.of(filteredSchema.nodeSchema(), filteredNodes.idMap(), filteredNodes.propertyStores())) .relationshipImportResult(RelationshipImportResult.of(filteredRelationships)) .concurrency(config.concurrency()) .build(); From a816af890fe2d11487df140417bac6352f7bb27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 19 Jan 2023 12:41:51 +0100 Subject: [PATCH 076/400] Pass correct node schema in arrow NodeImporter Co-authored-by: Martin Junghanns --- .../java/org/neo4j/gds/core/loading/Nodes.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index c0b8748dfcc..aa4d851c05a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -86,22 +86,6 @@ static Nodes of( return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); } - static Nodes of(IdMap idMap, Map properties, PropertyState propertyState) { - NodePropertyStore.Builder builder = NodePropertyStore.builder(); - properties.forEach((mapping, nodeProperties) -> builder.putProperty( - mapping.propertyKey(), - NodeProperty.of( - mapping.propertyKey(), - propertyState, - nodeProperties, - mapping.defaultValue().isUserDefined() - ? mapping.defaultValue() - : nodeProperties.valueType().fallbackValue() - ) - )); - return ImmutableNodes.of(NodeSchema.empty(), idMap, builder.build()); - } - static Nodes of(NodeSchema nodeSchema, IdMap idMap, Map properties, PropertyState propertyState) { NodePropertyStore.Builder builder = NodePropertyStore.builder(); properties.forEach((mapping, nodeProperties) -> builder.putProperty( From 53eb8998c3934a6009e5c94c2953027487d41eaf Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 19 Jan 2023 14:24:13 +0100 Subject: [PATCH 077/400] Add test for merging duplicate property keys Co-Authored-By: Paul Horn --- .../NodeLabelTokenToPropertyKeysTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java index a41e544706c..b8415c4a15d 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -140,5 +140,25 @@ void shouldFailForIncompatibleTypes() { .hasMessageContaining("Incompatible value types between input schema and input data.") .hasMessageContaining("['foo']"); } + + @Test + void testDuplicatePropertyKeys() { + var mapping = NodeLabelTokenToPropertyKeys.lazy(); + mapping.add(NodeLabelTokens.ofStrings("A", "B"), List.of("foo", "bar")); + mapping.add(NodeLabelTokens.ofStrings("A"), List.of("foo")); + mapping.add(NodeLabelTokens.ofStrings("B"), List.of("bar")); + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + ); + assertThat(mapping.propertySchemas(NodeLabel.of("A"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("B"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + )); + } } From 9cced6a0b7c2f8952f66726edb5cb2d586763fc2 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 19 Jan 2023 15:03:45 +0100 Subject: [PATCH 078/400] Infer default value from ValueType --- .../core/loading/IndexPropertyMappings.java | 28 ++++++------------- .../org/neo4j/gds/core/loading/Nodes.java | 3 +- .../core/loading/ScanningNodesImporter.java | 12 ++++---- .../org/neo4j/gds/api/DefaultValueTest.java | 28 +++++++++++++++++-- .../java/org/neo4j/gds/api/DefaultValue.java | 17 +++++++++++ 5 files changed, 59 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java b/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java index 537ea6fbe11..54cc9001789 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/IndexPropertyMappings.java @@ -49,25 +49,7 @@ // TODO: should be named LoadablePropertyMappings final class IndexPropertyMappings { - static LoadablePropertyMappings prepareProperties( - GraphProjectFromStoreConfig graphProjectConfig, - GraphDimensions graphDimensions, - TransactionContext transaction - ) { - Map storeLoadedProperties = graphProjectConfig - .nodeProjections() - .projections() - .entrySet() - .stream() - .collect(toMap( - Map.Entry::getKey, - entry -> entry.getValue().properties() - )); - - return prepareLoadableProperties(graphDimensions, transaction, storeLoadedProperties); - } - - static Map propertyMappingsByLabel(GraphProjectFromStoreConfig graphProjectConfig) { + static Map propertyMappings(GraphProjectFromStoreConfig graphProjectConfig) { return graphProjectConfig .nodeProjections() .projections() @@ -81,6 +63,14 @@ static Map propertyMappingsByLabel(GraphProjectFrom )); } + static LoadablePropertyMappings prepareProperties( + GraphProjectFromStoreConfig graphProjectConfig, + GraphDimensions graphDimensions, + TransactionContext transaction + ) { + return prepareLoadableProperties(graphDimensions, transaction, propertyMappings(graphProjectConfig)); + } + private static LoadablePropertyMappings prepareLoadableProperties( GraphDimensions dimensions, TransactionContext transaction, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index aa4d851c05a..6ba34d5abdf 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -25,6 +25,7 @@ import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; @@ -73,7 +74,7 @@ static Nodes of( var propertySchema = ImmutablePropertySchema.builder() .key(propertyMapping.propertyKey()) .valueType(nodePropertyValues.valueType()) - .defaultValue(propertyMapping.defaultValue()) + .defaultValue(DefaultValue.of(nodePropertyValues.valueType())) .state(propertyState) .build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 2cf3ae84170..819909d4453 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -90,16 +90,16 @@ public static ScanningNodesImporter scanningNodesImporter( ); } - var propertyMappings = IndexPropertyMappings.prepareProperties( + var propertyMappings = IndexPropertyMappings.propertyMappings(graphProjectConfig); + + var loadablePropertyMappings = IndexPropertyMappings.prepareProperties( graphProjectConfig, dimensions, loadingContext.transactionContext() ); - var propertyMappingsByLabel = IndexPropertyMappings.propertyMappingsByLabel(graphProjectConfig); - var nodePropertyImporter = initializeNodePropertyImporter( - propertyMappings, + loadablePropertyMappings, dimensions, concurrency ); @@ -111,7 +111,7 @@ public static ScanningNodesImporter scanningNodesImporter( progressTracker, concurrency, propertyMappings, - propertyMappingsByLabel, + loadablePropertyMappings, nodePropertyImporter, idMapBuilder, labelInformationBuilder @@ -124,8 +124,8 @@ private ScanningNodesImporter( GraphDimensions dimensions, ProgressTracker progressTracker, int concurrency, - IndexPropertyMappings.LoadablePropertyMappings propertyMappings, Map propertyMappingsByLabel, + IndexPropertyMappings.LoadablePropertyMappings propertyMappings, @Nullable NativeNodePropertyImporter nodePropertyImporter, IdMapBuilder idMapBuilder, LabelInformation.Builder labelInformationBuilder diff --git a/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java b/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java index 769fba4c0a0..616963b079a 100644 --- a/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java +++ b/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java @@ -180,14 +180,36 @@ void createUserDefinedValueTypes(Object valueToSet, Function fn assertThat(fn.apply(defaultValue)).isEqualTo(expectedValue); } + @Test + void initFromValueType() { + assertThat(DefaultValue.of(ValueType.DOUBLE)).isEqualTo(DefaultValue.forDouble()); + assertThat(DefaultValue.of(ValueType.LONG)).isEqualTo(DefaultValue.forLong()); + assertThat(DefaultValue.of(ValueType.LONG_ARRAY)).isEqualTo(DefaultValue.forLongArray()); + assertThat(DefaultValue.of(ValueType.FLOAT_ARRAY)).isEqualTo(DefaultValue.forFloatArray()); + assertThat(DefaultValue.of(ValueType.DOUBLE_ARRAY)).isEqualTo(DefaultValue.forDoubleArray()); + assertThat(DefaultValue.of(ValueType.STRING)).isEqualTo(DefaultValue.of(DEFAULT)); + } + private static Stream values() { return Stream.of( Arguments.of(42, (Function) DefaultValue::longValue, 42L), Arguments.of(42, (Function) DefaultValue::doubleValue, 42D), Arguments.of(13.37, (Function) DefaultValue::doubleValue, 13.37D), - Arguments.of(List.of(13.37, 42), (Function) DefaultValue::doubleArrayValue, new double[] {13.37D, 42D}), - Arguments.of(List.of(1337L, 42L), (Function) DefaultValue::longArrayValue, new long[] {1337L, 42L}), - Arguments.of(List.of(1337, 42), (Function) DefaultValue::longArrayValue, new long[] {1337L, 42L}) + Arguments.of( + List.of(13.37, 42), + (Function) DefaultValue::doubleArrayValue, + new double[]{13.37D, 42D} + ), + Arguments.of( + List.of(1337L, 42L), + (Function) DefaultValue::longArrayValue, + new long[]{1337L, 42L} + ), + Arguments.of( + List.of(1337, 42), + (Function) DefaultValue::longArrayValue, + new long[]{1337L, 42L} + ) ); } diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java index f954b848ea4..fa4018167f8 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java @@ -84,6 +84,23 @@ public static DefaultValue of(@Nullable Object defaultValue, ValueType type, boo } } + public static DefaultValue of(ValueType type) { + switch (type) { + case LONG: + return DefaultValue.forLong(); + case DOUBLE: + return DefaultValue.forDouble(); + case DOUBLE_ARRAY: + return DefaultValue.forDoubleArray(); + case FLOAT_ARRAY: + return DefaultValue.forFloatArray(); + case LONG_ARRAY: + return DefaultValue.forLongArray(); + default: + return DefaultValue.of(type.fallbackValue()); + } + } + private static DefaultValue ofFallBackValue(@Nullable Object defaultValue) { return of(defaultValue, false); } From 5f1938281beb06a6249adb5b40b6f96beb34b9de Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 19 Jan 2023 16:17:39 +0100 Subject: [PATCH 079/400] Consider case without label, but properties --- .../neo4j/gds/api/CSRGraphStoreFactory.java | 5 +- .../neo4j/gds/core/loading/CypherFactory.java | 4 +- .../NodeLabelTokenToPropertyKeys.java | 29 +++++++++-- .../loading/construction/NodesBuilder.java | 14 +++++- .../NodeLabelTokenToPropertyKeysTest.java | 49 +++++++++++++++++++ 5 files changed, 90 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java index eb5eb3a36ce..739e58aa2c5 100644 --- a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java @@ -42,10 +42,7 @@ public CSRGraphStoreFactory( super(graphProjectConfig, capabilities, loadingContext, dimensions); } - protected CSRGraphStore createGraphStore( - Nodes nodes, - RelationshipImportResult relationshipImportResult - ) { + protected CSRGraphStore createGraphStore(Nodes nodes, RelationshipImportResult relationshipImportResult) { return new GraphStoreBuilder() .databaseId(DatabaseId.of(loadingContext.graphDatabaseService())) .capabilities(capabilities) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index 554775323cd..4653d96dcc6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -148,7 +148,7 @@ public CSRGraphStore build() { progressTracker ).load(ktx.internalTransaction()); - var relationshipsAndProperties = new CypherRelationshipLoader( + var relationshipImportResult = new CypherRelationshipLoader( relationshipQuery(), nodes.idMap(), cypherConfig, @@ -158,7 +158,7 @@ public CSRGraphStore build() { var graphStore = createGraphStore( nodes, - relationshipsAndProperties + relationshipImportResult ); progressTracker.endSubTask("Loading"); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java index f57af989da7..4d9b703f5f6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import java.util.stream.IntStream; abstract class NodeLabelTokenToPropertyKeys { @@ -57,6 +58,11 @@ static NodeLabelTokenToPropertyKeys fixed(NodeSchema nodeSchema) { */ abstract void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys); + /** + * Returns all node labels in this mapping. + */ + abstract Set nodeLabels(); + /** * Return the property schemas for the given node label. */ @@ -78,6 +84,11 @@ void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys) { // silence is golden } + @Override + Set nodeLabels() { + return nodeSchema.availableLabels(); + } + @Override Map propertySchemas( NodeLabel nodeLabel, @@ -120,7 +131,6 @@ Map propertySchemas( ); } - return userDefinedPropertySchemas; } } @@ -140,7 +150,16 @@ void add(NodeLabelToken nodeLabelToken, Iterable propertyKeys) { propertyKeys.forEach(keys::add); return keys; }); + } + @Override + Set nodeLabels() { + return labelToPropertyKeys + .keySet() + .stream() + .map(nodeLabelToken -> nodeLabelToken.isEmpty() ? NodeLabelTokens.ofNodeLabel(NodeLabel.ALL_NODES) : nodeLabelToken) + .flatMap(nodeLabelToken -> IntStream.range(0, nodeLabelToken.size()).mapToObj(nodeLabelToken::get)) + .collect(Collectors.toSet()); } @Override @@ -150,7 +169,11 @@ Map propertySchemas( ) { return labelToPropertyKeys.keySet().stream() .filter(nodeLabelToken -> { - for (int i = 0; i < nodeLabelToken.size(); i++) { + if (nodeLabelToken.isEmpty() && nodeLabel == NodeLabel.ALL_NODES) { + return true; + } + var nodeLabelTokenCount = nodeLabelToken.size(); + for (int i = 0; i < nodeLabelTokenCount; i++) { if (nodeLabelToken.get(i).equals(nodeLabel)) { return true; } @@ -162,7 +185,7 @@ Map propertySchemas( propertyKey -> propertyKey, importPropertySchemas::get, (lhs, rhs) -> lhs - )); + )); } } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 372a3c10063..3694c216ed1 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -50,6 +50,7 @@ import org.neo4j.values.virtual.MapValue; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -223,9 +224,18 @@ private NodeSchema buildNodeSchema(IdMap idMap, Map nodePr .stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().propertySchema())); - idMap.availableNodeLabels().forEach(label -> { - nodeSchema.addLabel(label, this.nodeLabelTokenToPropertyKeys.propertySchemas(label, importPropertySchemas)); + // consider node labels without properties + var nodeLabels = new HashSet<>(idMap.availableNodeLabels()); + // and also node labels with associated properties + nodeLabels.addAll(nodeLabelTokenToPropertyKeys.nodeLabels()); + + nodeLabels.forEach(nodeLabel -> { + nodeSchema.addLabel( + nodeLabel, + this.nodeLabelTokenToPropertyKeys.propertySchemas(nodeLabel, importPropertySchemas) + ); }); + return nodeSchema; } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java index b8415c4a15d..9e43db597d9 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -160,5 +160,54 @@ void testDuplicatePropertyKeys() { "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) )); } + + @Test + void testEmptyNodeLabelTokenResolvesToAllNodes() { + var mapping = NodeLabelTokenToPropertyKeys.lazy(); + mapping.add(NodeLabelTokens.empty(), List.of("foo", "bar")); + + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + ); + + assertThat(mapping.propertySchemas(NodeLabel.ALL_NODES, importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + )); + } + + @Test + void testNodeLabelsLazily() { + var mapping = NodeLabelTokenToPropertyKeys.lazy(); + mapping.add(NodeLabelTokens.empty(), List.of("foo")); + mapping.add(NodeLabelTokens.of("A"), List.of("bar")); + mapping.add(NodeLabelTokens.ofStrings("A", "B"), List.of("baz")); + mapping.add(NodeLabelTokens.ofStrings("B", "C"), List.of("buz")); + + assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder( + NodeLabel.ALL_NODES, + NodeLabel.of("A"), + NodeLabel.of("B"), + NodeLabel.of("C") + ); + } + + @Test + void testNodeLabelsFixed() { + var mapping = NodeLabelTokenToPropertyKeys.fixed(NodeSchema.empty() + .addLabel(NodeLabel.ALL_NODES) + .addLabel(NodeLabel.of("A")) + .addLabel(NodeLabel.of("B")) + .addLabel(NodeLabel.of("C")) + ); + + assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder( + NodeLabel.ALL_NODES, + NodeLabel.of("A"), + NodeLabel.of("B"), + NodeLabel.of("C") + ); + } } From 95b506e27bf862062fcaef7e2eff3cd65f7df5b2 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 19 Jan 2023 16:42:42 +0100 Subject: [PATCH 080/400] Use default value user override or infer from value type --- .../main/java/org/neo4j/gds/core/loading/Nodes.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 6ba34d5abdf..3e6f615f0aa 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -71,15 +71,23 @@ static Nodes of( } else { propertyMappings.mappings().forEach(propertyMapping -> { var nodePropertyValues = properties.get(propertyMapping); + // The default value is either overridden by the user + // or inferred from the actual property value. + var defaultValue = propertyMapping.defaultValue().equals(DefaultValue.DEFAULT) + ? DefaultValue.of(nodePropertyValues.valueType()) + : propertyMapping.defaultValue(); var propertySchema = ImmutablePropertySchema.builder() .key(propertyMapping.propertyKey()) .valueType(nodePropertyValues.valueType()) - .defaultValue(DefaultValue.of(nodePropertyValues.valueType())) + .defaultValue(defaultValue) .state(propertyState) .build(); nodeSchema.addProperty(nodeLabel, propertySchema.key(), propertySchema); - nodePropertyStoreBuilder.putProperty(propertySchema.key(), ImmutableNodeProperty.of(nodePropertyValues, propertySchema)); + nodePropertyStoreBuilder.putProperty( + propertySchema.key(), + ImmutableNodeProperty.of(nodePropertyValues, propertySchema) + ); }); } })); From ea6e34a1aae3046e46b14cfaaac1b0ab5146df15 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Fri, 20 Jan 2023 09:03:52 +0100 Subject: [PATCH 081/400] Fix node schema construction in Arrow GDS import --- .../java/org/neo4j/gds/core/loading/Nodes.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 3e6f615f0aa..8624437d77c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -95,9 +95,14 @@ static Nodes of( return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); } - static Nodes of(NodeSchema nodeSchema, IdMap idMap, Map properties, PropertyState propertyState) { - NodePropertyStore.Builder builder = NodePropertyStore.builder(); - properties.forEach((mapping, nodeProperties) -> builder.putProperty( + static Nodes of( + NodeSchema nodeSchema, + IdMap idMap, + Map properties, + PropertyState propertyState + ) { + var propertyStoreBuilder = NodePropertyStore.builder(); + properties.forEach((mapping, nodeProperties) -> propertyStoreBuilder.putProperty( mapping.propertyKey(), NodeProperty.of( mapping.propertyKey(), @@ -108,6 +113,7 @@ static Nodes of(NodeSchema nodeSchema, IdMap idMap, Map Date: Fri, 20 Jan 2023 09:49:37 +0100 Subject: [PATCH 082/400] Clean up GDLFactory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../java/org/neo4j/gds/gdl/GdlFactory.java | 57 +------------------ 1 file changed, 1 insertion(+), 56 deletions(-) diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index 5ea833439fa..dee8d5333f9 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -22,7 +22,6 @@ import org.immutables.builder.Builder; import org.jetbrains.annotations.NotNull; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.CSRGraphStoreFactory; import org.neo4j.gds.api.DatabaseId; @@ -31,7 +30,6 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.api.schema.RelationshipSchema; @@ -49,7 +47,6 @@ import org.neo4j.gds.core.loading.construction.NodeLabelTokens; import org.neo4j.gds.core.loading.construction.PropertyValues; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; -import org.neo4j.gds.core.loading.nodeproperties.NodePropertiesFromStoreBuilder; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -165,34 +162,6 @@ protected ProgressTracker initProgressTracker() { @Override protected GraphSchema computeGraphSchema(Nodes nodes, RelationshipImportResult relationshipImportResult) { -// var nodeProperties = nodes.properties(); -// var nodeSchema = NodeSchema.empty(); -// gdlHandler -// .getVertices() -// .forEach(vertex -> { -// var labels = vertex.getLabels().stream().map(NodeLabel::of).collect(Collectors.toList()); -// if (labels.isEmpty()) { -// labels = List.of(NodeLabel.ALL_NODES); -// } -// -// labels.forEach(label -> vertex -// .getProperties() -// .forEach((propertyKey, propertyValue) -> nodeSchema -// .getOrCreateLabel(label) -// .addProperty( -// propertyKey, -// PropertySchema.of( -// propertyKey, -// nodeProperties.get(propertyKey).valueType(), -// nodeProperties.get(propertyKey).defaultValue(), -// nodeProperties.get(propertyKey).propertyState() -// ) -// ) -// )); -// }); -// // in case there were no properties add all labels -// nodes.idMap().availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); - var relationshipSchema = relationshipImportResult.importResults().entrySet().stream().reduce( RelationshipSchema.empty(), (unionSchema, entry) -> { @@ -259,31 +228,7 @@ private Nodes loadNodes() { ); }); - var nodes = nodesBuilder.build(); - return Nodes.of(nodes.schema(), nodes.idMap(), loadNodeProperties(nodes.idMap()), graphProjectConfig.propertyState()); - } - - private Map loadNodeProperties(IdMap idMap) { - var propertyBuilders = new HashMap(); - - gdlHandler.getVertices().forEach(vertex -> vertex - .getProperties() - .forEach((propertyKey, propertyValue) -> { - if (propertyValue instanceof List) { - propertyValue = convertListProperty((List) propertyValue); - } - - propertyBuilders.computeIfAbsent(PropertyMapping.of(propertyKey), (key) -> - NodePropertiesFromStoreBuilder.of( - DefaultValue.DEFAULT, - 1 - )).set(vertex.getId(), Values.of(propertyValue)); - })); - - return propertyBuilders - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().build(idMap))); + return nodesBuilder.build(); } @NotNull From 64ef5f21f0358b222e9fcc0c6b241e5f53540d0a Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Fri, 20 Jan 2023 10:05:24 +0100 Subject: [PATCH 083/400] Inline static factory methods from Nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../org/neo4j/gds/core/loading/Nodes.java | 70 ------------------- .../core/loading/ScanningNodesImporter.java | 36 +++++++++- .../org/neo4j/gds/api/DefaultValueTest.java | 10 --- .../java/org/neo4j/gds/api/DefaultValue.java | 17 ----- 4 files changed, 35 insertions(+), 98 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 8624437d77c..8e95632b3c0 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -21,22 +21,11 @@ import org.immutables.value.Value; import org.jetbrains.annotations.TestOnly; -import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; -import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; -import org.neo4j.gds.api.schema.ImmutablePropertySchema; import org.neo4j.gds.api.schema.NodeSchema; -import java.util.Map; - @ValueClass public interface Nodes { @@ -57,63 +46,4 @@ static Nodes of(IdMap idmap) { return ImmutableNodes.of(NodeSchema.empty(), idmap, NodePropertyStore.empty()); } - static Nodes of( - IdMap idMap, - Map propertyMappingsByLabel, - Map properties, - PropertyState propertyState - ) { - var nodeSchema = NodeSchema.empty(); - var nodePropertyStoreBuilder = NodePropertyStore.builder(); - propertyMappingsByLabel.forEach(((nodeLabel, propertyMappings) -> { - if (propertyMappings.mappings().isEmpty()) { - nodeSchema.addLabel(nodeLabel); - } else { - propertyMappings.mappings().forEach(propertyMapping -> { - var nodePropertyValues = properties.get(propertyMapping); - // The default value is either overridden by the user - // or inferred from the actual property value. - var defaultValue = propertyMapping.defaultValue().equals(DefaultValue.DEFAULT) - ? DefaultValue.of(nodePropertyValues.valueType()) - : propertyMapping.defaultValue(); - var propertySchema = ImmutablePropertySchema.builder() - .key(propertyMapping.propertyKey()) - .valueType(nodePropertyValues.valueType()) - .defaultValue(defaultValue) - .state(propertyState) - .build(); - - nodeSchema.addProperty(nodeLabel, propertySchema.key(), propertySchema); - nodePropertyStoreBuilder.putProperty( - propertySchema.key(), - ImmutableNodeProperty.of(nodePropertyValues, propertySchema) - ); - }); - } - })); - - return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); - } - - static Nodes of( - NodeSchema nodeSchema, - IdMap idMap, - Map properties, - PropertyState propertyState - ) { - var propertyStoreBuilder = NodePropertyStore.builder(); - properties.forEach((mapping, nodeProperties) -> propertyStoreBuilder.putProperty( - mapping.propertyKey(), - NodeProperty.of( - mapping.propertyKey(), - propertyState, - nodeProperties, - mapping.defaultValue().isUserDefined() - ? mapping.defaultValue() - : nodeProperties.valueType().fallbackValue() - ) - )); - - return ImmutableNodes.of(nodeSchema, idMap, propertyStoreBuilder.build()); - } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 819909d4453..93b5f0d18b8 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -27,7 +27,11 @@ import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; +import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.schema.ImmutablePropertySchema; +import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; @@ -199,7 +203,37 @@ public Nodes build() { importPropertiesFromIndex(idMap, nodeProperties); } - return Nodes.of(idMap, propertyMappingsByLabel, nodeProperties, PropertyState.PERSISTENT); + var nodeSchema = NodeSchema.empty(); + var nodePropertyStoreBuilder = NodePropertyStore.builder(); + + this.propertyMappingsByLabel.forEach(((nodeLabel, mappings) -> { + if (mappings.mappings().isEmpty()) { + nodeSchema.addLabel(nodeLabel); + } else { + mappings.mappings().forEach(propertyMapping -> { + var nodePropertyValues = nodeProperties.get(propertyMapping); + // The default value is either overridden by the user + // or inferred from the actual property value. + var defaultValue = propertyMapping.defaultValue().isUserDefined() + ? propertyMapping.defaultValue() + : nodePropertyValues.valueType().fallbackValue(); + var propertySchema = ImmutablePropertySchema.builder() + .key(propertyMapping.propertyKey()) + .valueType(nodePropertyValues.valueType()) + .defaultValue(defaultValue) + .state(PropertyState.PERSISTENT) + .build(); + + nodeSchema.addProperty(nodeLabel, propertySchema.key(), propertySchema); + nodePropertyStoreBuilder.putProperty( + propertySchema.key(), + ImmutableNodeProperty.of(nodePropertyValues, propertySchema) + ); + }); + } + })); + + return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); } private void importPropertiesFromIndex( diff --git a/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java b/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java index 616963b079a..f2ec1e3bdf3 100644 --- a/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java +++ b/core/src/test/java/org/neo4j/gds/api/DefaultValueTest.java @@ -180,16 +180,6 @@ void createUserDefinedValueTypes(Object valueToSet, Function fn assertThat(fn.apply(defaultValue)).isEqualTo(expectedValue); } - @Test - void initFromValueType() { - assertThat(DefaultValue.of(ValueType.DOUBLE)).isEqualTo(DefaultValue.forDouble()); - assertThat(DefaultValue.of(ValueType.LONG)).isEqualTo(DefaultValue.forLong()); - assertThat(DefaultValue.of(ValueType.LONG_ARRAY)).isEqualTo(DefaultValue.forLongArray()); - assertThat(DefaultValue.of(ValueType.FLOAT_ARRAY)).isEqualTo(DefaultValue.forFloatArray()); - assertThat(DefaultValue.of(ValueType.DOUBLE_ARRAY)).isEqualTo(DefaultValue.forDoubleArray()); - assertThat(DefaultValue.of(ValueType.STRING)).isEqualTo(DefaultValue.of(DEFAULT)); - } - private static Stream values() { return Stream.of( Arguments.of(42, (Function) DefaultValue::longValue, 42L), diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java index fa4018167f8..f954b848ea4 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/api/DefaultValue.java @@ -84,23 +84,6 @@ public static DefaultValue of(@Nullable Object defaultValue, ValueType type, boo } } - public static DefaultValue of(ValueType type) { - switch (type) { - case LONG: - return DefaultValue.forLong(); - case DOUBLE: - return DefaultValue.forDouble(); - case DOUBLE_ARRAY: - return DefaultValue.forDoubleArray(); - case FLOAT_ARRAY: - return DefaultValue.forFloatArray(); - case LONG_ARRAY: - return DefaultValue.forLongArray(); - default: - return DefaultValue.of(type.fallbackValue()); - } - } - private static DefaultValue ofFallBackValue(@Nullable Object defaultValue) { return of(defaultValue, false); } From 05ff6db0f2918a593a4969c09ca60fa7401b908a Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Fri, 20 Jan 2023 10:08:11 +0100 Subject: [PATCH 084/400] Make Nodes#schema mandatory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../main/java/org/neo4j/gds/core/loading/Nodes.java | 11 +---------- .../java/org/neo4j/gds/config/WriteConfigTest.java | 10 ++++++++-- .../org/neo4j/gds/projection/GraphAggregator.java | 2 +- .../io/file/GraphStoreRelationshipVisitorTest.java | 9 +++++++-- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 8e95632b3c0..af4f6bd6671 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.loading; import org.immutables.value.Value; -import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; @@ -29,10 +28,7 @@ @ValueClass public interface Nodes { - @Value.Default - default NodeSchema schema() { - return NodeSchema.empty(); - } + NodeSchema schema(); IdMap idMap(); @@ -41,9 +37,4 @@ default NodePropertyStore properties() { return NodePropertyStore.empty(); } - @TestOnly - static Nodes of(IdMap idmap) { - return ImmutableNodes.of(NodeSchema.empty(), idmap, NodePropertyStore.empty()); - } - } diff --git a/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java b/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java index 42de1d59d32..2ddf4329989 100644 --- a/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java @@ -24,11 +24,12 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.loading.GraphStoreBuilder; +import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; -import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import java.util.List; @@ -44,11 +45,16 @@ void validateGraphStoreCapabilities(boolean isBackedByDatabase) { var config = CypherMapWrapper.empty(); var testConfig = new TestWriteConfigImpl(config); + var nodes = ImmutableNodes.builder() + .idMap(new DirectIdMap(0)) + .schema(NodeSchema.empty()) + .build(); + var testGraphStore = new GraphStoreBuilder() .databaseId(DatabaseId.from("neo4j")) .capabilities(ImmutableStaticCapabilities.of(isBackedByDatabase)) .schema(GraphSchema.empty()) - .nodes(Nodes.of(new DirectIdMap(0))) + .nodes(nodes) .relationshipImportResult(RelationshipImportResult.of(Map.of())) .concurrency(1) .build(); diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index ababb4a834e..efa52bd893d 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -652,7 +652,7 @@ private AdjacencyCompressor.ValueMapper buildNodesWithProperties(GraphStoreBuild ); }); - graphStoreBuilder.nodes(nodesBuilder.build()); + graphStoreBuilder.nodes(nodesBuilder.schema(nodeSchema).build()); // Relationships are added using their intermediate node ids. // In order to map to the final internal ids, we need to use diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java index e6ba8e67ffe..a7f5363c28a 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java @@ -27,8 +27,8 @@ import org.neo4j.gds.api.RelationshipPropertyStore; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; import org.neo4j.gds.core.loading.GraphStoreBuilder; +import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; -import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; @@ -151,11 +151,16 @@ private Graph createGraph( var actualRelationships = FileToGraphStoreImporter.relationshipTopologyAndProperties(relationshipBuildersByType); assertThat(actualRelationships.importedRelationships()).isEqualTo(expectedImportedRelationshipsCount); + var nodes = ImmutableNodes.builder() + .idMap(expectedGraph) + .schema(expectedGraph.schema().nodeSchema()) + .build(); + Map propertyStores = actualRelationships.properties(); return new GraphStoreBuilder() .schema(expectedGraph.schema()) .capabilities(ImmutableStaticCapabilities.of(true)) - .nodes(Nodes.of(expectedGraph)) + .nodes(nodes) .relationshipImportResult(RelationshipImportResult.of( actualRelationships.topologies(), propertyStores, From cebc1152765bd733fa1d16337233d9e1470bc5c4 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Fri, 20 Jan 2023 10:31:23 +0100 Subject: [PATCH 085/400] Align Nodes building in Scanning and Arrow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../org/neo4j/gds/core/loading/Nodes.java | 47 +++++++++++++++++++ .../core/loading/ScanningNodesImporter.java | 36 +------------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index af4f6bd6671..c56be2ebc1c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -20,11 +20,20 @@ package org.neo4j.gds.core.loading; import org.immutables.value.Value; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.PropertyMapping; +import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; +import org.neo4j.gds.api.properties.nodes.NodePropertyValues; +import org.neo4j.gds.api.schema.ImmutablePropertySchema; import org.neo4j.gds.api.schema.NodeSchema; +import java.util.Map; + @ValueClass public interface Nodes { @@ -37,4 +46,42 @@ default NodePropertyStore properties() { return NodePropertyStore.empty(); } + static Nodes of( + IdMap idMap, + Map propertyMappings, + Map propertyValues, + PropertyState propertyState + ) { + var nodeSchema = NodeSchema.empty(); + var nodePropertyStoreBuilder = NodePropertyStore.builder(); + + propertyMappings.forEach(((nodeLabel, mappings) -> { + if (mappings.mappings().isEmpty()) { + nodeSchema.addLabel(nodeLabel); + } else { + mappings.mappings().forEach(propertyMapping -> { + var nodePropertyValues = propertyValues.get(propertyMapping); + // The default value is either overridden by the user + // or inferred from the actual property value. + var defaultValue = propertyMapping.defaultValue().isUserDefined() + ? propertyMapping.defaultValue() + : nodePropertyValues.valueType().fallbackValue(); + var propertySchema = ImmutablePropertySchema.builder() + .key(propertyMapping.propertyKey()) + .valueType(nodePropertyValues.valueType()) + .defaultValue(defaultValue) + .state(propertyState) + .build(); + + nodeSchema.addProperty(nodeLabel, propertySchema.key(), propertySchema); + nodePropertyStoreBuilder.putProperty( + propertySchema.key(), + ImmutableNodeProperty.of(nodePropertyValues, propertySchema) + ); + }); + } + })); + + return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); + } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 93b5f0d18b8..36f6a09d71a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -27,11 +27,7 @@ import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; -import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; -import org.neo4j.gds.api.schema.ImmutablePropertySchema; -import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; @@ -203,37 +199,7 @@ public Nodes build() { importPropertiesFromIndex(idMap, nodeProperties); } - var nodeSchema = NodeSchema.empty(); - var nodePropertyStoreBuilder = NodePropertyStore.builder(); - - this.propertyMappingsByLabel.forEach(((nodeLabel, mappings) -> { - if (mappings.mappings().isEmpty()) { - nodeSchema.addLabel(nodeLabel); - } else { - mappings.mappings().forEach(propertyMapping -> { - var nodePropertyValues = nodeProperties.get(propertyMapping); - // The default value is either overridden by the user - // or inferred from the actual property value. - var defaultValue = propertyMapping.defaultValue().isUserDefined() - ? propertyMapping.defaultValue() - : nodePropertyValues.valueType().fallbackValue(); - var propertySchema = ImmutablePropertySchema.builder() - .key(propertyMapping.propertyKey()) - .valueType(nodePropertyValues.valueType()) - .defaultValue(defaultValue) - .state(PropertyState.PERSISTENT) - .build(); - - nodeSchema.addProperty(nodeLabel, propertySchema.key(), propertySchema); - nodePropertyStoreBuilder.putProperty( - propertySchema.key(), - ImmutableNodeProperty.of(nodePropertyValues, propertySchema) - ); - }); - } - })); - - return ImmutableNodes.of(nodeSchema, idMap, nodePropertyStoreBuilder.build()); + return Nodes.of(idMap, this.propertyMappingsByLabel, nodeProperties, PropertyState.PERSISTENT); } private void importPropertiesFromIndex( From 4c4dbf8206cd33974864aa43d53a7c633a7886bd Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Fri, 20 Jan 2023 11:53:44 +0100 Subject: [PATCH 086/400] Use Nodes directly in FileToGraphStoreImporter --- .../io/file/FileToGraphStoreImporter.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 8955203911d..00f9af3a861 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -32,10 +32,9 @@ import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.io.GraphStoreGraphPropertyVisitor; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; -import org.neo4j.gds.core.loading.CSRGraphStoreUtil; import org.neo4j.gds.core.loading.GraphStoreBuilder; -import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; +import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodesBuilder; @@ -138,11 +137,11 @@ private void importGraphStore(FileInput fileInput) { graphStoreBuilder.capabilities(fileInput.capabilities()); var nodes = importNodes(fileInput); - importRelationships(fileInput, nodes); + importRelationships(fileInput, nodes.idMap()); importGraphProperties(fileInput); } - private IdMap importNodes(FileInput fileInput) { + private Nodes importNodes(FileInput fileInput) { progressTracker.beginSubTask(); NodeSchema nodeSchema = fileInput.nodeSchema(); graphSchemaBuilder.nodeSchema(nodeSchema); @@ -165,19 +164,12 @@ private IdMap importNodes(FileInput fileInput) { ParallelUtil.run(tasks, Pools.DEFAULT); var nodes = nodesBuilder.build(); - var nodeImportResultBuilder = ImmutableNodes.builder().idMap(nodes.idMap()); - var schemaProperties = nodeSchema.unionProperties(); - CSRGraphStoreUtil.extractNodeProperties( - nodeImportResultBuilder, - schemaProperties::get, - nodes.properties().propertyValues() - ); + this.graphStoreBuilder.nodes(nodes); - graphStoreBuilder.nodes(nodeImportResultBuilder.build()); + this.progressTracker.endSubTask(); - progressTracker.endSubTask(); - return nodes.idMap(); + return nodes; } private void importRelationships(FileInput fileInput, IdMap nodes) { From 0f76bbaae141bbf41b1501f7420197726c3b013b Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Fri, 20 Jan 2023 14:29:15 +0100 Subject: [PATCH 087/400] Extend node property mapping validation to default values --- .../linkprediction-pipelines/training.adoc | 2 +- .../neo4j/gds/AbstractNodeProjections.java | 39 ++++++++--- .../org/neo4j/gds/NodeProjectionsTest.java | 68 ++++++++++++++----- 3 files changed, 80 insertions(+), 29 deletions(-) diff --git a/doc/modules/ROOT/pages/machine-learning/linkprediction-pipelines/training.adoc b/doc/modules/ROOT/pages/machine-learning/linkprediction-pipelines/training.adoc index ffe836b2107..eb1373392d0 100644 --- a/doc/modules/ROOT/pages/machine-learning/linkprediction-pipelines/training.adoc +++ b/doc/modules/ROOT/pages/machine-learning/linkprediction-pipelines/training.adoc @@ -312,7 +312,7 @@ CALL gds.graph.project( 'fullGraph', { Person: { - properties: ['age'] + properties: {age: {defaultValue: 1}} }, City: { properties: {age: {defaultValue: 1}} diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java b/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java index 98cb3ea5a9c..bc585203be6 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Objects; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -175,23 +176,39 @@ private static void validateIdentifierName(String identifier) { @Value.Check public void validatePropertyKeyMappings() { - var mapping = new HashMap(); + var seenMappings = new HashMap(); projections().values().stream() .flatMap(nodeProjection -> nodeProjection.properties().stream()) .forEach(propertyMapping -> { var propertyKey = propertyMapping.propertyKey(); - var neoKey = propertyMapping.neoPropertyKey(); - - if (mapping.containsKey(propertyKey) && !mapping.get(propertyKey).equals(neoKey)) { - throw new IllegalArgumentException(formatWithLocale( - "Specifying multiple neoPropertyKeys for the same property is not allowed, found propertyKey: %s, neoPropertyKeys: %s, %s.", - propertyKey, - neoKey, - mapping.get(propertyKey) - )); + + if (seenMappings.containsKey(propertyKey)) { + // we have another mapping with the same GDS key + var seenMapping = seenMappings.get(propertyKey); + + if (!Objects.equals(seenMapping.neoPropertyKey(), propertyMapping.neoPropertyKey())) { + throw new IllegalArgumentException(formatWithLocale( + "Specifying multiple neoPropertyKeys for the same property is not allowed, " + + "found propertyKey: `%s` with conflicting neoPropertyKeys: `%s`, `%s`.", + propertyKey, + propertyMapping.neoPropertyKey(), + seenMapping.neoPropertyKey() + )); + } + + if (!Objects.equals(seenMapping.defaultValue(), propertyMapping.defaultValue())) { + throw new IllegalArgumentException(formatWithLocale( + "Specifying different default values for the same property with identical neoPropertyKey is not allowed, " + + "found propertyKey: `%s` with conflicting default values: `%s`, `%s`.", + propertyKey, + propertyMapping.defaultValue().getObject(), + seenMapping.defaultValue().getObject() + )); + } } - mapping.put(propertyKey, neoKey); + + seenMappings.put(propertyKey, propertyMapping); }); } } diff --git a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java index 218ec70b04e..ec3f6e03c07 100644 --- a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java +++ b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java @@ -147,37 +147,71 @@ void shouldFailOnDuplicatePropertyKeys() { .hasMessage("Duplicate property key `prop`"); } - @Test - void shouldFailOnAmbiguousNodePropertyDefinition() { - var builder = NodeProjections.builder().projections(Map.of( + static Stream ambiguousPropertyMappings() { + return Stream.of( + Arguments.of( + "different neo key", + PropertyMapping.of("foo", "bar", DefaultValue.DEFAULT), + PropertyMapping.of("foo", "baz", DefaultValue.DEFAULT), + new String[]{ + "Specifying multiple neoPropertyKeys for the same property is not allowed, found propertyKey: `foo` with conflicting neoPropertyKeys:", + "`bar`", + "`baz`" + } + ), + Arguments.of( + "different default value types", + PropertyMapping.of("foo", "baz", DefaultValue.forLong()), + PropertyMapping.of("foo", "baz", DefaultValue.forDouble()), + new String[]{ + "Specifying different default values for the same property with identical neoPropertyKey is not allowed, found propertyKey: `foo` with conflicting default values:", + "`-9223372036854775808`", + "`NaN`", + } + ), + Arguments.of( + "different default values", + PropertyMapping.of("foo", "baz", DefaultValue.of(42)), + PropertyMapping.of("foo", "baz", DefaultValue.of(1337)), + new String[]{ + "Specifying different default values for the same property with identical neoPropertyKey is not allowed, found propertyKey: `foo` with conflicting default values:", + "`42`", + "`1337`", + } + ) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("ambiguousPropertyMappings") + void shouldFailOnAmbiguousNodePropertyDefinition( + String ignore, + PropertyMapping first, + PropertyMapping second, + String... messages + ) { + var builder = ImmutableNodeProjections.builder().projections(Map.of( NodeLabel.of("A"), NodeProjection .builder() .label("A") - .properties(PropertyMappings - .builder() - .addMapping(PropertyMapping.of("foo", "bar", DefaultValue.DEFAULT)) - .build() - ) + .addProperty(first) .build(), NodeLabel.of("B"), NodeProjection .builder() .label("B") - .properties(PropertyMappings - .builder() - .addMapping(PropertyMapping.of("foo", "baz", DefaultValue.DEFAULT)) - .build() - ) + .addProperty(second) .build() )); - assertThatThrownBy(builder::build) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Specifying multiple neoPropertyKeys for the same property is not allowed, found propertyKey: foo, neoPropertyKeys"); + var throwableAssert = assertThatThrownBy(builder::build) + .isInstanceOf(IllegalArgumentException.class); + for (var message : messages) { + throwableAssert.hasMessageContaining(message); + } } @Test From 27a754921fedde526a7843b40472103de70d22ee Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 23 Jan 2023 17:16:35 +0100 Subject: [PATCH 088/400] Add NodeLabelToken#nodeLabels convenience method Co-authored-by: Paul Horn --- .../loading/construction/NodeLabelToken.java | 10 +++++ .../NodeLabelTokenToPropertyKeys.java | 3 +- .../construction/NodeLabelTokensTest.java | 45 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokensTest.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelToken.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelToken.java index dfc70c6905d..80f2ac61bf7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelToken.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelToken.java @@ -22,6 +22,9 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.NodeLabel; +import java.util.stream.IntStream; +import java.util.stream.Stream; + public interface NodeLabelToken { /** @@ -46,6 +49,13 @@ public interface NodeLabelToken { @NotNull NodeLabel get(int index); String[] getStrings(); + + /** + * @return a stream of {@link org.neo4j.gds.NodeLabel}s represented by this token. + */ + default Stream nodeLabels() { + return IntStream.range(0, this.size()).mapToObj(this::get); + } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java index 4d9b703f5f6..d6f787c4c0c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -29,7 +29,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; -import java.util.stream.IntStream; abstract class NodeLabelTokenToPropertyKeys { @@ -158,7 +157,7 @@ Set nodeLabels() { .keySet() .stream() .map(nodeLabelToken -> nodeLabelToken.isEmpty() ? NodeLabelTokens.ofNodeLabel(NodeLabel.ALL_NODES) : nodeLabelToken) - .flatMap(nodeLabelToken -> IntStream.range(0, nodeLabelToken.size()).mapToObj(nodeLabelToken::get)) + .flatMap(NodeLabelToken::nodeLabels) .collect(Collectors.toSet()); } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokensTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokensTest.java new file mode 100644 index 00000000000..155cfa24b69 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokensTest.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.NodeLabel; + +import static org.assertj.core.api.Assertions.assertThat; + +class NodeLabelTokensTest { + + @Test + void testStream() { + var aLabel = NodeLabel.of("A"); + var bLabel = NodeLabel.of("B"); + var cLabel = NodeLabel.of("C"); + + assertThat(NodeLabelTokens.empty().nodeLabels()).isEmpty(); + assertThat(NodeLabelTokens.ofNodeLabel(aLabel).nodeLabels()).contains(aLabel); + assertThat( + NodeLabelTokens.ofNodeLabels(aLabel, bLabel, cLabel).nodeLabels() + ).contains(aLabel, bLabel, cLabel); + assertThat( + NodeLabelTokens.ofStrings(aLabel.name(), bLabel.name(), cLabel.name()).nodeLabels() + ).contains(aLabel, bLabel, cLabel); + } + +} From 6f19c342505a8fefb9bf54c10a05b3185f063bd1 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 24 Jan 2023 10:57:28 +0100 Subject: [PATCH 089/400] Adapt to still existing @DataClass --- .../src/test/java/org/neo4j/gds/NodeProjectionsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java index ec3f6e03c07..48b4a4ceb74 100644 --- a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java +++ b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java @@ -190,7 +190,7 @@ void shouldFailOnAmbiguousNodePropertyDefinition( PropertyMapping second, String... messages ) { - var builder = ImmutableNodeProjections.builder().projections(Map.of( + var builder = NodeProjections.builder().projections(Map.of( NodeLabel.of("A"), NodeProjection .builder() From 10bd6bce34b4346fd807e22f573689b9b0523a2e Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 23 Jan 2023 18:31:36 +0100 Subject: [PATCH 090/400] Validate that sizeInBytes is >= 0 to avoid UnsupportedOperationException --- .../main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java index 11fc8d4d9f2..2310df211e8 100644 --- a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java @@ -58,8 +58,10 @@ protected CSRGraphStore createGraphStore( protected void logLoadingSummary(GraphStore graphStore) { var sizeInBytes = MemoryUsage.sizeOf(graphStore); - var memoryUsage = MemoryUsage.humanReadable(sizeInBytes); - progressTracker.logInfo(formatWithLocale("Actual memory usage of the loaded graph: %s", memoryUsage)); + if (sizeInBytes >= 0) { + var memoryUsage = MemoryUsage.humanReadable(sizeInBytes); + progressTracker.logInfo(formatWithLocale("Actual memory usage of the loaded graph: %s", memoryUsage)); + } } protected abstract GraphSchema computeGraphSchema( From 24bf084ae0420699c31439fb7891ab07ed4ec48e Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Tue, 24 Jan 2023 11:13:50 +0100 Subject: [PATCH 091/400] 1337-ify PR --- .../NodeLabelTokenToPropertyKeysTest.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java index 9e43db597d9..b2574012c06 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -195,19 +195,9 @@ void testNodeLabelsLazily() { @Test void testNodeLabelsFixed() { - var mapping = NodeLabelTokenToPropertyKeys.fixed(NodeSchema.empty() - .addLabel(NodeLabel.ALL_NODES) - .addLabel(NodeLabel.of("A")) - .addLabel(NodeLabel.of("B")) - .addLabel(NodeLabel.of("C")) - ); + var mapping = NodeLabelTokenToPropertyKeys.fixed(NodeSchema.empty().addLabel(NodeLabel.ALL_NODES).addLabel(NodeLabel.of("A")).addLabel(NodeLabel.of("B")).addLabel(NodeLabel.of("C"))); - assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder( - NodeLabel.ALL_NODES, - NodeLabel.of("A"), - NodeLabel.of("B"), - NodeLabel.of("C") - ); + assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder(NodeLabel.ALL_NODES, NodeLabel.of("A"), NodeLabel.of("B"), NodeLabel.of("C")); } } From 56d2ea29b71c35246bab419ca50e413548e57f0c Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 23 Jan 2023 12:08:25 +0100 Subject: [PATCH 092/400] Make property schema building thread local MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../loading/construction/GraphFactory.java | 4 +- .../NodeLabelTokenToPropertyKeys.java | 25 +++++++++++-- .../loading/construction/NodesBuilder.java | 37 ++++++++++++++----- .../NodeLabelTokenToPropertyKeysTest.java | 24 ++++++++++++ 4 files changed, 75 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index b0da169f8b7..1e297e31e14 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -121,7 +121,7 @@ static NodesBuilder nodesBuilder( maxOriginalId, threadCount, TokenToNodeLabel.lazy(), - NodeLabelTokenToPropertyKeys.lazy(), + NodeLabelTokenToPropertyKeys::lazy, new ConcurrentHashMap<>(), idMapBuilder, labelInformation, @@ -162,7 +162,7 @@ private static NodesBuilder fromSchema( maxOriginalId, concurrency, TokenToNodeLabel.fixed(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping), - NodeLabelTokenToPropertyKeys.fixed(nodeSchema), + () -> NodeLabelTokenToPropertyKeys.fixed(nodeSchema), new ConcurrentHashMap<>(propertyBuildersByPropertyKey), idMapBuilder, hasLabelInformation, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java index d6f787c4c0c..b8dcf6cb90d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -24,10 +24,10 @@ import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.utils.StringJoining; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; abstract class NodeLabelTokenToPropertyKeys { @@ -70,6 +70,25 @@ abstract Map propertySchemas( Map importPropertySchemas ); + static NodeLabelTokenToPropertyKeys merge( + NodeLabelTokenToPropertyKeys left, + NodeLabelTokenToPropertyKeys right, + Map importPropertySchemas + ) { + var merge = NodeLabelTokenToPropertyKeys.lazy(); + + left.nodeLabels().forEach(nodeLabel -> { + var propertyKeys = left.propertySchemas(nodeLabel, importPropertySchemas).keySet(); + merge.add(NodeLabelTokens.ofNodeLabel(nodeLabel), propertyKeys); + }); + right.nodeLabels().forEach(nodeLabel -> { + var propertyKeys = right.propertySchemas(nodeLabel, importPropertySchemas).keySet(); + merge.add(NodeLabelTokens.ofNodeLabel(nodeLabel), propertyKeys); + }); + + return merge; + } + private static class Fixed extends NodeLabelTokenToPropertyKeys { private final NodeSchema nodeSchema; @@ -136,10 +155,10 @@ Map propertySchemas( private static class Lazy extends NodeLabelTokenToPropertyKeys { - private final ConcurrentHashMap> labelToPropertyKeys; + private final Map> labelToPropertyKeys; Lazy() { - this.labelToPropertyKeys = new ConcurrentHashMap<>(); + this.labelToPropertyKeys = new HashMap<>(); } @Override diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 3694c216ed1..ddaa9080554 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -50,6 +50,7 @@ import org.neo4j.values.virtual.MapValue; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -58,6 +59,7 @@ import java.util.concurrent.atomic.LongAdder; import java.util.function.Function; import java.util.function.LongPredicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.stream.Collectors.toMap; @@ -81,13 +83,11 @@ public final class NodesBuilder { private final ConcurrentMap propertyBuildersByPropertyKey; - private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; - NodesBuilder( long maxOriginalId, int concurrency, TokenToNodeLabel tokenToNodeLabel, - NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, + Supplier nodeLabelTokenToPropertyKeysSupplier, ConcurrentMap propertyBuildersByPropertyKey, IdMapBuilder idMapBuilder, boolean hasLabelInformation, @@ -97,7 +97,6 @@ public final class NodesBuilder { ) { this.maxOriginalId = maxOriginalId; this.concurrency = concurrency; - this.nodeLabelTokenToPropertyKeys = nodeLabelTokenToPropertyKeys; this.idMapBuilder = idMapBuilder; this.propertyStates = propertyStates; this.labelInformationBuilder = !hasLabelInformation @@ -128,7 +127,7 @@ public final class NodesBuilder { hasLabelInformation, hasProperties, tokenToNodeLabel, - nodeLabelTokenToPropertyKeys, + nodeLabelTokenToPropertyKeysSupplier.get(), propertyBuilderFn ) ); @@ -202,12 +201,15 @@ public Nodes build() { public Nodes build(long highestNeoId) { // Flush remaining buffer contents this.threadLocalBuilder.forEach(ThreadLocalBuilder::flush); + // Collect token to property keys for final merge + var nodeLabelTokenToPropertyKeysList = new ArrayList(); + this.threadLocalBuilder.forEach(tlb -> nodeLabelTokenToPropertyKeysList.add(tlb.nodeLabelTokenToPropertyKeys)); // Clean up resources held by local builders this.threadLocalBuilder.close(); var idMap = this.idMapBuilder.build(labelInformationBuilder, highestNeoId, concurrency); var nodeProperties = buildProperties(idMap); - var nodeSchema = buildNodeSchema(idMap, nodeProperties); + var nodeSchema = buildNodeSchema(idMap, nodeLabelTokenToPropertyKeysList, nodeProperties); var nodePropertyStore = NodePropertyStore.builder().properties(nodeProperties).build(); return ImmutableNodes.builder() @@ -217,7 +219,11 @@ public Nodes build(long highestNeoId) { .build(); } - private NodeSchema buildNodeSchema(IdMap idMap, Map nodeProperties) { + private NodeSchema buildNodeSchema( + IdMap idMap, + Collection localNodeLabelTokenToPropertyKeys, + Map nodeProperties + ) { var nodeSchema = NodeSchema.empty(); var importPropertySchemas = nodeProperties .entrySet() @@ -227,12 +233,19 @@ private NodeSchema buildNodeSchema(IdMap idMap, Map nodePr // consider node labels without properties var nodeLabels = new HashSet<>(idMap.availableNodeLabels()); // and also node labels with associated properties - nodeLabels.addAll(nodeLabelTokenToPropertyKeys.nodeLabels()); - + localNodeLabelTokenToPropertyKeys.forEach(mapping -> nodeLabels.addAll(mapping.nodeLabels())); + // merge into a global mapping + var globalNodeLabelTokenToPropertyKeys = localNodeLabelTokenToPropertyKeys + .stream() + .reduce( + NodeLabelTokenToPropertyKeys.lazy(), + (left, right) -> NodeLabelTokenToPropertyKeys.merge(left, right, importPropertySchemas) + ); + // and construct final node property schema nodeLabels.forEach(nodeLabel -> { nodeSchema.addLabel( nodeLabel, - this.nodeLabelTokenToPropertyKeys.propertySchemas(nodeLabel, importPropertySchemas) + globalNodeLabelTokenToPropertyKeys.propertySchemas(nodeLabel, importPropertySchemas) ); }); @@ -331,6 +344,10 @@ private static class ThreadLocalBuilder implements AutoCloseable { this.batchNodeProperties = new ArrayList<>(buffer.capacity()); } + NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys() { + return this.nodeLabelTokenToPropertyKeys; + } + public void addNode(long originalId, NodeLabelToken nodeLabels) { if (!seenNodeIdPredicate.test(originalId)) { long[] labels = getOrCreateLabelTokens(nodeLabels); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java index b2574012c06..91e8f2fa785 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -199,5 +199,29 @@ void testNodeLabelsFixed() { assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder(NodeLabel.ALL_NODES, NodeLabel.of("A"), NodeLabel.of("B"), NodeLabel.of("C")); } + + @Test + void testMerge() { + var importPropertySchemas = Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + ); + + var left = NodeLabelTokenToPropertyKeys.lazy(); + left.add(NodeLabelTokens.ofStrings("A"), List.of("foo")); + + var right = NodeLabelTokenToPropertyKeys.lazy(); + right.add(NodeLabelTokens.ofStrings("B"), List.of("bar")); + + var mapping = NodeLabelTokenToPropertyKeys.merge(left, right, importPropertySchemas); + + assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder(NodeLabel.of("A"), NodeLabel.of("B")); + assertThat(mapping.propertySchemas(NodeLabel.of("A"), importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + )); + assertThat(mapping.propertySchemas(NodeLabel.of("B"), importPropertySchemas)).isEqualTo(Map.of( + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + )); + } } From 5165902a7dcc49dc7c4c9c4e3d8505a8023b6c1c Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Mon, 23 Jan 2023 13:54:17 +0100 Subject: [PATCH 093/400] Improve tests and refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../NodeLabelTokenToPropertyKeys.java | 14 ++-- .../loading/construction/NodesBuilder.java | 68 ++++++++++--------- .../NodeLabelTokenToPropertyKeysTest.java | 36 +++++++--- 3 files changed, 72 insertions(+), 46 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java index b8dcf6cb90d..0603ec2308f 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeys.java @@ -70,23 +70,27 @@ abstract Map propertySchemas( Map importPropertySchemas ); - static NodeLabelTokenToPropertyKeys merge( + /** + * Computes the union of the two given mappings without + * changing the contents of the mappings themselves. + */ + static NodeLabelTokenToPropertyKeys union( NodeLabelTokenToPropertyKeys left, NodeLabelTokenToPropertyKeys right, Map importPropertySchemas ) { - var merge = NodeLabelTokenToPropertyKeys.lazy(); + var union = NodeLabelTokenToPropertyKeys.lazy(); left.nodeLabels().forEach(nodeLabel -> { var propertyKeys = left.propertySchemas(nodeLabel, importPropertySchemas).keySet(); - merge.add(NodeLabelTokens.ofNodeLabel(nodeLabel), propertyKeys); + union.add(NodeLabelTokens.ofNodeLabel(nodeLabel), propertyKeys); }); right.nodeLabels().forEach(nodeLabel -> { var propertyKeys = right.propertySchemas(nodeLabel, importPropertySchemas).keySet(); - merge.add(NodeLabelTokens.ofNodeLabel(nodeLabel), propertyKeys); + union.add(NodeLabelTokens.ofNodeLabel(nodeLabel), propertyKeys); }); - return merge; + return union; } private static class Fixed extends NodeLabelTokenToPropertyKeys { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index ddaa9080554..003710fbefb 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -199,17 +199,11 @@ public Nodes build() { } public Nodes build(long highestNeoId) { - // Flush remaining buffer contents - this.threadLocalBuilder.forEach(ThreadLocalBuilder::flush); - // Collect token to property keys for final merge - var nodeLabelTokenToPropertyKeysList = new ArrayList(); - this.threadLocalBuilder.forEach(tlb -> nodeLabelTokenToPropertyKeysList.add(tlb.nodeLabelTokenToPropertyKeys)); - // Clean up resources held by local builders - this.threadLocalBuilder.close(); + var localLabelTokenToPropertyKeys = closeThreadLocalBuilders(); var idMap = this.idMapBuilder.build(labelInformationBuilder, highestNeoId, concurrency); var nodeProperties = buildProperties(idMap); - var nodeSchema = buildNodeSchema(idMap, nodeLabelTokenToPropertyKeysList, nodeProperties); + var nodeSchema = buildNodeSchema(idMap, localLabelTokenToPropertyKeys, nodeProperties); var nodePropertyStore = NodePropertyStore.builder().properties(nodeProperties).build(); return ImmutableNodes.builder() @@ -219,37 +213,53 @@ public Nodes build(long highestNeoId) { .build(); } + private List closeThreadLocalBuilders() { + // Flush remaining buffer contents + this.threadLocalBuilder.forEach(ThreadLocalBuilder::flush); + // Collect token to property keys for final union + var labelTokenToPropertyKeys = new ArrayList(); + this.threadLocalBuilder.forEach(tlb -> labelTokenToPropertyKeys.add(tlb.nodeLabelTokenToPropertyKeys)); + // Clean up resources held by local builders + this.threadLocalBuilder.close(); + + return labelTokenToPropertyKeys; + } + private NodeSchema buildNodeSchema( IdMap idMap, - Collection localNodeLabelTokenToPropertyKeys, + Collection localLabelTokenToPropertyKeys, Map nodeProperties ) { - var nodeSchema = NodeSchema.empty(); - var importPropertySchemas = nodeProperties + + // Collect the property schemas from the imported property values. + var propertyKeysToSchema = nodeProperties .entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().propertySchema())); - - // consider node labels without properties - var nodeLabels = new HashSet<>(idMap.availableNodeLabels()); - // and also node labels with associated properties - localNodeLabelTokenToPropertyKeys.forEach(mapping -> nodeLabels.addAll(mapping.nodeLabels())); - // merge into a global mapping - var globalNodeLabelTokenToPropertyKeys = localNodeLabelTokenToPropertyKeys + // Union the label to property key mappings from each import thread. + var globalLabelTokenToPropertyKeys = localLabelTokenToPropertyKeys .stream() .reduce( NodeLabelTokenToPropertyKeys.lazy(), - (left, right) -> NodeLabelTokenToPropertyKeys.merge(left, right, importPropertySchemas) - ); - // and construct final node property schema - nodeLabels.forEach(nodeLabel -> { - nodeSchema.addLabel( - nodeLabel, - globalNodeLabelTokenToPropertyKeys.propertySchemas(nodeLabel, importPropertySchemas) + (left, right) -> NodeLabelTokenToPropertyKeys.union(left, right, propertyKeysToSchema) ); - }); + // Collect node labels without properties from the id map + // as they are not stored in the above union mapping. + var nodeLabels = new HashSet<>(idMap.availableNodeLabels()); + // Add labels that actually have node properties attached. + localLabelTokenToPropertyKeys.forEach(localMapping -> nodeLabels.addAll(localMapping.nodeLabels())); - return nodeSchema; + // Use all labels and the global label to property + // key mapping to construct the final node schema. + return nodeLabels.stream() + .reduce( + NodeSchema.empty(), + (unionSchema, nodeLabel) -> unionSchema.addLabel( + nodeLabel, + globalLabelTokenToPropertyKeys.propertySchemas(nodeLabel, propertyKeysToSchema) + ), + (lhs, rhs) -> lhs + ); } private Map buildProperties(IdMap idMap) { @@ -344,10 +354,6 @@ private static class ThreadLocalBuilder implements AutoCloseable { this.batchNodeProperties = new ArrayList<>(buffer.capacity()); } - NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys() { - return this.nodeLabelTokenToPropertyKeys; - } - public void addNode(long originalId, NodeLabelToken nodeLabels) { if (!seenNodeIdPredicate.test(originalId)) { long[] labels = getOrCreateLabelTokens(nodeLabels); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java index 91e8f2fa785..ef844f9c34b 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -204,24 +204,40 @@ void testNodeLabelsFixed() { void testMerge() { var importPropertySchemas = Map.of( "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), - "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG_ARRAY, DefaultValue.forLongArray(), PropertyState.PERSISTENT) ); - var left = NodeLabelTokenToPropertyKeys.lazy(); - left.add(NodeLabelTokens.ofStrings("A"), List.of("foo")); + var aLabel = NodeLabel.of("A"); + var bLabel = NodeLabel.of("B"); + var cLabel = NodeLabel.of("C"); - var right = NodeLabelTokenToPropertyKeys.lazy(); - right.add(NodeLabelTokens.ofStrings("B"), List.of("bar")); + var mapping0 = NodeLabelTokenToPropertyKeys.lazy(); + mapping0.add(NodeLabelTokens.ofNodeLabels(aLabel), List.of("foo")); + mapping0.add(NodeLabelTokens.ofNodeLabels(cLabel), List.of("bar")); - var mapping = NodeLabelTokenToPropertyKeys.merge(left, right, importPropertySchemas); + var mapping1 = NodeLabelTokenToPropertyKeys.lazy(); + mapping1.add(NodeLabelTokens.ofNodeLabels(bLabel), List.of("bar")); - assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder(NodeLabel.of("A"), NodeLabel.of("B")); - assertThat(mapping.propertySchemas(NodeLabel.of("A"), importPropertySchemas)).isEqualTo(Map.of( - "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT) + var mapping2 = NodeLabelTokenToPropertyKeys.lazy(); + mapping2.add(NodeLabelTokens.ofNodeLabels(aLabel, cLabel), List.of("baz")); + + var union = NodeLabelTokenToPropertyKeys.union(mapping0, mapping1, importPropertySchemas); + union = NodeLabelTokenToPropertyKeys.union(union, mapping2, importPropertySchemas); + + assertThat(union.nodeLabels()).containsExactlyInAnyOrder(aLabel, bLabel, cLabel); + + assertThat(union.propertySchemas(aLabel, importPropertySchemas)).isEqualTo(Map.of( + "foo", PropertySchema.of("foo", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT), + "baz", PropertySchema.of("baz", ValueType.LONG_ARRAY, DefaultValue.forLongArray(), PropertyState.PERSISTENT) )); - assertThat(mapping.propertySchemas(NodeLabel.of("B"), importPropertySchemas)).isEqualTo(Map.of( + assertThat(union.propertySchemas(bLabel, importPropertySchemas)).isEqualTo(Map.of( "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT) )); + assertThat(union.propertySchemas(cLabel, importPropertySchemas)).isEqualTo(Map.of( + "bar", PropertySchema.of("bar", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.PERSISTENT), + "baz", PropertySchema.of("baz", ValueType.LONG_ARRAY, DefaultValue.forLongArray(), PropertyState.PERSISTENT) + )); } } From ada925fab79d8d5c30cfe51ea8a00c3820b3c1f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 24 Jan 2023 15:35:19 +0100 Subject: [PATCH 094/400] Link from appendix to specific pages --- .../ROOT/pages/graph-catalog-relationship-ops.adoc | 1 + .../additional-operation-references.adoc | 4 ++-- .../operations-reference/graph-operation-references.adoc | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc b/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc index bfa36e83220..ee502d00e13 100644 --- a/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc +++ b/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc @@ -661,6 +661,7 @@ ORDER BY source ASC, target ASC NOTE: The properties we want to stream must exist for each specified relationship type. +[[catalog-graph-relationship-to-undirected-example]] === Convert to undirected Some algorithms such as Triangle Count and Link Prediction expect undirected relationships. The following shows how to convert the relationships of type `LIKES` in the graph from directed to undirected by creating an undirected relationship of new type `INTERACTS`. diff --git a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc index b9d3e745dac..72aedc27dc1 100644 --- a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc @@ -23,8 +23,8 @@ | xref:common-usage/debug-sysinfo.adoc[Status of the system] | `gds.debug.sysInfo` | xref:management-ops/create-cypher-db.adoc[Create an impermanent database backed by a projected graph] | `gds.alpha.create.cypherdb` | xref:common-usage/monitoring-system.adoc[Get an overview of the system's workload and available resources] | `gds.alpha.systemMonitor` -| Back-up graphs and models to disk | `gds.alpha.backup` -| Restore persisted graphs and models to memory | `gds.alpha.restore` +| xref:management-ops/backup-restore.adoc[Back-up graphs and models to disk] | `gds.alpha.backup` +| xref:management-ops/backup-restore.adoc[Restore persisted graphs and models to memory] | `gds.alpha.restore` | xref:production-deployment/defaults-and-limits.adoc[List configured defaults] | `gds.alpha.config.defaults.list` | xref:production-deployment/defaults-and-limits.adoc[Configure a default] | `gds.alpha.config.defaults.set` | xref:production-deployment/defaults-and-limits.adoc#_limits_on_configuration_values[List configured limits] | `gds.alpha.config.limits.list` diff --git a/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc index c9465dbca24..6388133bb9d 100644 --- a/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/graph-operation-references.adoc @@ -59,7 +59,7 @@ | `gds.beta.graph.export.csv` | `gds.beta.graph.export.csv.estimate` |xref:graph-catalog-relationship-ops.adoc#catalog-graph-stream-relationship-topology-example[Stream relationship topologies to the procedure caller] | `gds.beta.graph.relationships.stream` -.2+<.^|Convert directed relationships to undirected +.2+<.^|xref:graph-catalog-relationship-ops.adoc#catalog-graph-relationship-to-undirected-example[Convert directed relationships to undirected] | `gds.beta.graph.relationships.toUndirected` | `gds.beta.graph.relationships.toUndirected.estimate` @@ -75,7 +75,7 @@ |Description | Operation |Drop a graph property from a named graph | `gds.alpha.graph.graphProperty.drop` |Stream a graph property to the procedure caller | `gds.alpha.graph.graphProperty.stream` -|Sample a subgraph using random walk with restarts | `gds.alpha.graph.sample.rwr` -|Add node labels to the in-memory graph | `gds.alpha.graph.nodeLabel.mutate` -|Write node labels to the database | `gds.alpha.graph.nodeLabel.write` +|xref:management-ops/projections/rwr.adoc[Sample a subgraph using random walk with restarts] | `gds.alpha.graph.sample.rwr` +|xref:graph-catalog-node-ops.adoc#catalog-graph-mutate-node-label-example[Add node labels to the in-memory graph] | `gds.alpha.graph.nodeLabel.mutate` +|xref:graph-catalog-node-ops.adoc#catalog-graph-write-node-label-example[Write node labels to the database] | `gds.alpha.graph.nodeLabel.write` |=== From a9c3d8b89c03a7c5a033e5906d6432b1636ca92e Mon Sep 17 00:00:00 2001 From: Pontus Melke Date: Thu, 12 Jan 2023 22:21:31 +0100 Subject: [PATCH 095/400] Attempt to reduce contention - Use get + getAndSet instead of just getAndSet - Use one AtomicBoolean instead of two --- .../neo4j/gds/projection/GraphAggregator.java | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index efa52bd893d..bba3b1c7467 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -157,6 +157,7 @@ void projectNextRelationship( NodeLabelToken sourceNodeLabels = NodeLabelTokens.missing(); NodeLabelToken targetNodeLabels = NodeLabelTokens.missing(); + this.configValidator.validateConfigs(nodesConfig, relationshipConfig); if (nodesConfig instanceof MapValue) { sourceNodePropertyValues = GraphImporter.propertiesConfig("sourceNodeProperties", (MapValue) nodesConfig); sourceNodeLabels = labelsConfig("sourceNodeLabels", (MapValue) nodesConfig); @@ -168,8 +169,6 @@ void projectNextRelationship( ); targetNodeLabels = labelsConfig("targetNodeLabels", (MapValue) nodesConfig); } - - this.configValidator.validateNodesConfig((MapValue) nodesConfig); } var data = initGraphData( @@ -309,18 +308,27 @@ private static final class ConfigValidator { "relationshipType" ); - private final AtomicBoolean validateNodes = new AtomicBoolean(true); - private final AtomicBoolean validateRelationships = new AtomicBoolean(true); - - void validateNodesConfig(MapValue nodesConfig) { - if (this.validateNodes.getAndSet(false)) { - ConfigKeyValidation.requireOnlyKeysFrom(NODES_CONFIG_KEYS, nodesConfig.keySet()); - } - } - void validateRelationshipsConfig(MapValue relationshipConfig) { - if (this.validateRelationships.getAndSet(false)) { - ConfigKeyValidation.requireOnlyKeysFrom(RELATIONSHIPS_CONFIG_KEYS, relationshipConfig.keySet()); + private final AtomicBoolean validate = new AtomicBoolean(true); + + void validateConfigs(AnyValue nodesConfig, AnyValue relationshipConfig) { + if (nodesConfig instanceof MapValue || relationshipConfig instanceof MapValue) { + if (this.validate.get()) { + if (this.validate.getAndSet(false)) { + if (nodesConfig instanceof MapValue) { + ConfigKeyValidation.requireOnlyKeysFrom( + NODES_CONFIG_KEYS, + ((MapValue) nodesConfig).keySet() + ); + } + if (relationshipConfig instanceof MapValue) { + ConfigKeyValidation.requireOnlyKeysFrom( + NODES_CONFIG_KEYS, + ((MapValue) relationshipConfig).keySet() + ); + } + } + } } } } @@ -465,8 +473,6 @@ void update( if (relationshipConfig instanceof MapValue) { relationshipProperties = propertiesConfig("properties", (MapValue) relationshipConfig); relationshipType = typeConfig("relationshipType", (MapValue) relationshipConfig); - - configValidator.validateRelationshipsConfig((MapValue) relationshipConfig); } var intermediateSourceId = loadNode(sourceNode, sourceNodeLabels, sourceNodePropertyValues); @@ -723,6 +729,7 @@ static MapValue propertiesConfig( @NotNull MapValue propertiesConfig ) { var nodeProperties = propertiesConfig.get(propertyKey); + if (nodeProperties instanceof MapValue) { return (MapValue) nodeProperties; } From cb38708a19e5f6f88a75e8ac912a73015b5939d0 Mon Sep 17 00:00:00 2001 From: Pontus Melke Date: Fri, 13 Jan 2023 09:33:59 +0100 Subject: [PATCH 096/400] fix bug --- .../src/main/java/org/neo4j/gds/projection/GraphAggregator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index bba3b1c7467..3bd10655864 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -323,7 +323,7 @@ void validateConfigs(AnyValue nodesConfig, AnyValue relationshipConfig) { } if (relationshipConfig instanceof MapValue) { ConfigKeyValidation.requireOnlyKeysFrom( - NODES_CONFIG_KEYS, + RELATIONSHIPS_CONFIG_KEYS, ((MapValue) relationshipConfig).keySet() ); } From 77d1e013962a1209592d6e0341ed7083c32746b5 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 18 Jan 2023 14:49:28 +0100 Subject: [PATCH 097/400] Add test for non-uniform node properties/labels in Cypher Aggregation --- .../gds/projection/CypherAggregationTest.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index d10edf1cfd8..3b885e744b7 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -162,6 +162,53 @@ void testArbitraryIds() { } } + @Test + void testDifferentPropertySchemas() { + var query = "UNWIND [" + + " [0, 1, 'a', {}, 'rel', {}], " + + " [2, 3, 'b', {x:1}, 'rel2', {weight: 0.1}]," + + " [5, 6, 'c', {y:1}, 'rel3', {hq: 0.1}]" + + "] AS data" + + " RETURN gds.alpha.graph.project(" + + " 'g'," + + " data[0]," + + " data[1]," + + " {sourceNodeLabels: data[2], sourceNodeProperties: data[3]}," + + " {relationshipType: data[4], properties: data[5]}," + + " {}" + + ")"; + + runQuery(query); + + var graph = GraphStoreCatalog.get("", db.databaseName(), "g").graphStore().getUnion(); + + // these fail to separate the property schemas + assertThat(graph.schema().nodeSchema().get(NodeLabel.of("a")).properties().keySet()).isEmpty(); + assertThat(graph.schema().nodeSchema().get(NodeLabel.of("b")).properties().keySet()).containsExactly("x"); + assertThat(graph.schema().nodeSchema().get(NodeLabel.of("c")).properties().keySet()).containsExactly("y"); + + // these also fail like that, but if one of them has no props (like rel does) no properties are retained at all + // so 2 problems to fix + assertThat(graph + .schema() + .relationshipSchema() + .get(org.neo4j.gds.RelationshipType.of("rel")) + .properties() + .keySet()).isEmpty(); + assertThat(graph + .schema() + .relationshipSchema() + .get(org.neo4j.gds.RelationshipType.of("rel2")) + .properties() + .keySet()).containsExactly("weight"); + assertThat(graph + .schema() + .relationshipSchema() + .get(org.neo4j.gds.RelationshipType.of("rel3")) + .properties() + .keySet()).containsExactly("hq"); + } + @ParameterizedTest @CsvSource({"13.37, Double", "true, Boolean", "false, Boolean", "null, NO_VALUE", "\"42\", String", "[42], List", "[13.37], List", "{foo:42}, Map", "{foo:13.37}, Map"}) void testInvalidArbitraryIds(String idLiteral, String invalidType) { From d35742ef4c20b9ce1d979eb792ca2d0ff5bad493 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Thu, 19 Jan 2023 14:47:00 +0100 Subject: [PATCH 098/400] Use lazily built node schema in Cypher Aggregation --- .../gds/core/loading/LazyIdMapBuilder.java | 10 ++- .../neo4j/gds/projection/GraphAggregator.java | 85 +++---------------- 2 files changed, 21 insertions(+), 74 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index 937a9c712ce..57d682136a4 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -22,6 +22,7 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PartialIdMap; +import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -31,7 +32,6 @@ import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; import java.util.Map; -import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicBoolean; @@ -113,7 +113,11 @@ public interface HighLimitIdMapAndProperties { NodeSchema schema(); - Optional> nodeProperties(); + NodePropertyStore propertyStore(); + + default Map properties() { + return propertyStore().propertyValues(); + } } public HighLimitIdMapAndProperties build() { @@ -145,7 +149,7 @@ public OptionalLong rootNodeCount() { .idMap(idMap) .intermediateIdMap(partialIdMap) .schema(nodes.schema()) - .nodeProperties(nodes.properties().propertyValues()) + .propertyStore(nodes.properties()) .build(); } } diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index 3bd10655864..9b002870f6c 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -21,24 +21,18 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.DefaultValue; -import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.compat.CompatUserAggregator; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.ConfigKeyValidation; import org.neo4j.gds.core.compress.AdjacencyCompressor; -import org.neo4j.gds.core.loading.CSRGraphStoreUtil; import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.loading.ImmutableNodes; @@ -188,8 +182,7 @@ void projectNextRelationship( targetNodePropertyValues, sourceNodeLabels, targetNodeLabels, - relationshipConfig, - this.configValidator + relationshipConfig ); } @@ -334,7 +327,6 @@ void validateConfigs(AnyValue nodesConfig, AnyValue relationshipConfig) { } // Does the actual importing work once we can initialize it with the first row - @SuppressWarnings("CodeBlock2Expr") private static final class GraphImporter { private final String graphName; private final GraphProjectFromCypherAggregationConfig config; @@ -464,8 +456,7 @@ void update( @Nullable MapValue targetNodePropertyValues, NodeLabelToken sourceNodeLabels, NodeLabelToken targetNodeLabels, - AnyValue relationshipConfig, - ConfigValidator configValidator + AnyValue relationshipConfig ) { MapValue relationshipProperties = null; RelationshipType relationshipType = RelationshipType.ALL_RELATIONSHIPS; @@ -624,75 +615,27 @@ private static double[] loadMultipleRelationshipProperties( } private AdjacencyCompressor.ValueMapper buildNodesWithProperties(GraphStoreBuilder graphStoreBuilder) { - var idMapAndProperties = this.idMapBuilder.build(); - var nodes = idMapAndProperties.idMap(); - - var maybeNodeProperties = idMapAndProperties.nodeProperties(); - - var nodesBuilder = ImmutableNodes.builder().idMap(nodes); - - var nodePropertySchema = maybeNodeProperties - .map(nodeProperties -> nodeSchemaWithProperties( - nodes.availableNodeLabels(), - nodeProperties - )) - .orElseGet(() -> nodeSchemaWithoutProperties(nodes.availableNodeLabels())) - .unionProperties(); - - NodeSchema nodeSchema = NodeSchema.empty(); - nodes.availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); - nodePropertySchema.forEach((propertyKey, propertySchema) -> { - nodes.availableNodeLabels().forEach(label -> { - nodeSchema.getOrCreateLabel(label).addProperty(propertySchema.key(), propertySchema); - }); - }); + + var idMap = idMapAndProperties.idMap(); + var nodeSchema = idMapAndProperties.schema(); this.graphSchemaBuilder.nodeSchema(nodeSchema); - maybeNodeProperties.ifPresent(allNodeProperties -> { - CSRGraphStoreUtil.extractNodeProperties( - nodesBuilder, - nodePropertySchema::get, - allNodeProperties - ); - }); + var nodes = ImmutableNodes + .builder() + .idMap(idMap) + .schema(nodeSchema) + .properties(idMapAndProperties.propertyStore()) + .build(); - graphStoreBuilder.nodes(nodesBuilder.schema(nodeSchema).build()); + // graphStoreBuilder.nodes(nodesBuilder.schema(nodeSchema).build()); + graphStoreBuilder.nodes(nodes); // Relationships are added using their intermediate node ids. // In order to map to the final internal ids, we need to use // the mapping function of the wrapped id map. - return nodes.rootIdMap()::toMappedNodeId; - } - - private static NodeSchema nodeSchemaWithProperties( - Iterable nodeLabels, - Map propertyMap - ) { - var nodeSchema = NodeSchema.empty(); - - nodeLabels.forEach((nodeLabel) -> { - propertyMap.forEach((propertyName, nodeProperties) -> { - nodeSchema.getOrCreateLabel(nodeLabel).addProperty( - propertyName, - PropertySchema.of( - propertyName, - nodeProperties.valueType(), - nodeProperties.valueType().fallbackValue(), - PropertyState.TRANSIENT - ) - ); - }); - }); - - return nodeSchema; - } - - private static NodeSchema nodeSchemaWithoutProperties(Iterable nodeLabels) { - var nodeSchema = NodeSchema.empty(); - nodeLabels.forEach(nodeSchema::getOrCreateLabel); - return nodeSchema; + return idMap.rootIdMap()::toMappedNodeId; } private void buildRelationshipsWithProperties( From b0b12e55da8cfc1609aacba04d57a5f481ff4daa Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Fri, 20 Jan 2023 10:10:48 +0100 Subject: [PATCH 099/400] Initialize relationship schema per type --- .../loading/construction/GraphFactory.java | 7 +- .../neo4j/gds/projection/GraphAggregator.java | 98 ++++++++++--------- .../api/schema/RelationshipSchemaEntry.java | 2 +- 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index b0da169f8b7..3f2e9e12c3d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -222,9 +222,10 @@ static RelationshipsBuilder relationshipsBuilder( Optional validateRelationships, Optional concurrency, Optional indexInverse, - Optional executorService + Optional executorService, + Optional loadRelationshipProperties ) { - var loadRelationshipProperties = !propertyConfigs.isEmpty(); + var doLoadRelationshipProperties = loadRelationshipProperties.orElse(!propertyConfigs.isEmpty()); var aggregations = propertyConfigs.isEmpty() ? new Aggregation[]{aggregation.orElse(Aggregation.DEFAULT)} @@ -290,7 +291,7 @@ static RelationshipsBuilder relationshipsBuilder( .bufferSize(bufferSize) .propertyConfigs(propertyConfigs) .isMultiGraph(isMultiGraph) - .loadRelationshipProperty(loadRelationshipProperties) + .loadRelationshipProperty(doLoadRelationshipProperties) .direction(Direction.fromOrientation(actualOrientation)) .executorService(executorService.orElse(Pools.DEFAULT)) .concurrency(finalConcurrency); diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index 9b002870f6c..b9d58f25945 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -29,6 +29,7 @@ import org.neo4j.gds.api.schema.ImmutableGraphSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import org.neo4j.gds.compat.CompatUserAggregator; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.ConfigKeyValidation; @@ -41,6 +42,7 @@ import org.neo4j.gds.core.loading.ReadHelper; import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; +import org.neo4j.gds.core.loading.construction.ImmutablePropertyConfig; import org.neo4j.gds.core.loading.construction.NodeLabelToken; import org.neo4j.gds.core.loading.construction.NodeLabelTokens; import org.neo4j.gds.core.loading.construction.PropertyValues; @@ -59,6 +61,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -331,7 +334,7 @@ private static final class GraphImporter { private final String graphName; private final GraphProjectFromCypherAggregationConfig config; private final LazyIdMapBuilder idMapBuilder; - private final @Nullable List relationshipPropertySchemas; +// private final @Nullable List relationshipPropertySchemas; private final boolean canWriteToDatabase; private final ExtractNodeId extractNodeId; @@ -350,7 +353,7 @@ private GraphImporter( this.graphName = graphName; this.config = config; this.idMapBuilder = idMapBuilder; - this.relationshipPropertySchemas = relationshipPropertySchemas; +// this.relationshipPropertySchemas = relationshipPropertySchemas; this.canWriteToDatabase = canWriteToDatabase; this.lock = lock; this.relImporters = new ConcurrentHashMap<>(); @@ -469,23 +472,34 @@ void update( var intermediateSourceId = loadNode(sourceNode, sourceNodeLabels, sourceNodePropertyValues); if (targetNode != NoValue.NO_VALUE) { - var relImporter = this.relImporters.computeIfAbsent(relationshipType, this::newRelImporter); + RelationshipsBuilder relImporter; + // we do the check before to avoid having to create a new lambda instance on every call + if (this.relImporters.containsKey(relationshipType)) { + relImporter = this.relImporters.get(relationshipType); + } else { + var finalRelationshipProperties = relationshipProperties; + relImporter = this.relImporters.computeIfAbsent( + relationshipType, + type -> newRelImporter(type, finalRelationshipProperties) + ); + } + var intermediateTargetId = loadNode(targetNode, targetNodeLabels, targetNodePropertyValues); - if (this.relationshipPropertySchemas != null) { - assert relationshipProperties != null; - if (this.relationshipPropertySchemas.size() == 1) { - var relationshipProperty = this.relationshipPropertySchemas.get(0).key(); - double propertyValue = loadOneRelationshipProperty( - relationshipProperties, - relationshipProperty - ); - relImporter.addFromInternal(intermediateSourceId, intermediateTargetId, propertyValue); + if (relationshipProperties != null) { + if (relationshipProperties.size() == 1) { + relationshipProperties.foreach((key, value) -> { + var property = ReadHelper.extractValue(value, DefaultValue.DOUBLE_DEFAULT_FALLBACK); + relImporter.addFromInternal(intermediateSourceId, intermediateTargetId, property); + }); } else { - var propertyValues = loadMultipleRelationshipProperties( - relationshipProperties, - this.relationshipPropertySchemas - ); + var propertyValues = new double[relationshipProperties.size()]; + int[] index = {0}; + relationshipProperties.foreach((key, value) -> { + var property = ReadHelper.extractValue(value, DefaultValue.DOUBLE_DEFAULT_FALLBACK); + var i = index[0]++; + propertyValues[i] = property; + }); relImporter.addFromInternal(intermediateSourceId, intermediateTargetId, propertyValues); } } else { @@ -528,8 +542,7 @@ AggregationResult result(String username, DatabaseId databaseId, ProgressTimer t .build(); } - private RelationshipsBuilder newRelImporter(RelationshipType relType) { - + private RelationshipsBuilder newRelImporter(RelationshipType relType, @Nullable MapValue properties) { var undirectedTypes = this.config.undirectedRelationshipTypes(); var orientation = undirectedTypes.contains(relType.name) || undirectedTypes.contains("*") ? UNDIRECTED @@ -546,27 +559,12 @@ private RelationshipsBuilder newRelImporter(RelationshipType relType) { .indexInverse(indexInverse) .concurrency(this.config.readConcurrency()); - // There is a potential race between initializing the relationships builder and the - // relationship property schemas. Both happen under lock, but under different ones. - // Relationship builders are initialized as part of computeIfAbsent which uses the - // lock inside ConcurrentHashMap, while `this.relationshipPropertySchemas` is initialized - // using the lock in this class. - // - // We have to ensure that the property schemas field is fully initialized, before we - // create the relationships builder. This can only be achieved by using the same lock - // for both actions. This should not affect performance, as we are doing this inside of - // computeIfAbsent which is only called once. - this.lock.lock(); - try { - if (this.relationshipPropertySchemas != null) { - for (var relationshipPropertySchema : this.relationshipPropertySchemas) { - relationshipsBuilderBuilder.addPropertyConfig( - GraphFactory.PropertyConfig.of(relationshipPropertySchema.key()) - ); - } + if (properties != null) { + for (String propertyKey : properties.keySet()) { + relationshipsBuilderBuilder.addPropertyConfig( + ImmutablePropertyConfig.builder().propertyKey(propertyKey).build() + ); } - } finally { - this.lock.unlock(); } return relationshipsBuilderBuilder.build(); @@ -643,20 +641,27 @@ private void buildRelationshipsWithProperties( AdjacencyCompressor.ValueMapper valueMapper ) { var relationshipImportResultBuilder = RelationshipImportResult.builder(); - var relationshipSchemas = new ArrayList(); + var relationshipSchemas = new HashMap(); this.relImporters.forEach((relationshipType, relImporter) -> { var relationships = relImporter.build( Optional.of(valueMapper), Optional.empty() ); - relationshipSchemas.add(relationships.relationshipSchema(relationshipType)); + var schema = relationships.relationshipSchema(relationshipType); + relationshipSchemas.put(relationshipType, schema.get(relationshipType)); +// schema +// .entries() +// .forEach(entry -> relationshipSchemas.merge( +// relationshipType, +// entry, +// RelationshipSchemaEntry::union +// )); + relationshipImportResultBuilder.putImportResult(relationshipType, relationships); }); - var relationshipSchema = relationshipSchemas - .stream() - .reduce(RelationshipSchema.empty(), RelationshipSchema::union); + var relationshipSchema = new RelationshipSchema(relationshipSchemas); graphStoreBuilder.relationshipImportResult(relationshipImportResultBuilder.build()); this.graphSchemaBuilder.relationshipSchema(relationshipSchema); @@ -674,8 +679,13 @@ static MapValue propertiesConfig( var nodeProperties = propertiesConfig.get(propertyKey); if (nodeProperties instanceof MapValue) { - return (MapValue) nodeProperties; + var mapProperties = (MapValue) nodeProperties; + if (mapProperties.isEmpty()) { + return null; + } + return mapProperties; } + if (nodeProperties == NoValue.NO_VALUE) { return null; } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java index d468a96b722..2599cd6547e 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java @@ -65,7 +65,7 @@ boolean isUndirected() { } @Override - RelationshipSchemaEntry union(RelationshipSchemaEntry other) { + public RelationshipSchemaEntry union(RelationshipSchemaEntry other) { if (!other.identifier().equals(this.identifier())) { throw new UnsupportedOperationException( formatWithLocale( From 184d5c50b123051a88e69b81555df76630fef514 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Mon, 23 Jan 2023 15:29:40 +0100 Subject: [PATCH 100/400] Always say that we have properties and labels in CAgg --- .../main/java/org/neo4j/gds/projection/GraphAggregator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index b9d58f25945..0b202a71376 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -417,9 +417,7 @@ private static LazyIdMapBuilder idMapBuilder( @Nullable MapValue targetNodeProperties, int readConcurrency ) { - boolean hasLabelInformation = !(sourceNodeLabels.isMissing() && targetNodeLabels.isMissing()); - boolean hasProperties = !(sourceNodeProperties == null && targetNodeProperties == null); - return new LazyIdMapBuilder(readConcurrency, hasLabelInformation, hasProperties); + return new LazyIdMapBuilder(readConcurrency, true, true); } private static @Nullable List relationshipPropertySchemas(AnyValue relationshipConfigValue) { From 781328ec6ccd5e8e6f28e4f5bb3a5a1d0d41e218 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Mon, 23 Jan 2023 15:30:33 +0100 Subject: [PATCH 101/400] Remove now unused code for the initial lazy init schema --- .../neo4j/gds/projection/GraphAggregator.java | 137 ++---------------- 1 file changed, 11 insertions(+), 126 deletions(-) diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index 0b202a71376..14af0205f00 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -25,9 +25,7 @@ import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.DefaultValue; -import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import org.neo4j.gds.compat.CompatUserAggregator; @@ -59,8 +57,6 @@ import org.neo4j.values.virtual.MapValue; import org.neo4j.values.virtual.MapValueBuilder; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -149,6 +145,10 @@ void projectNextRelationship( AnyValue relationshipConfig, AnyValue config ) { + this.configValidator.validateConfigs(nodesConfig, relationshipConfig); + + var data = initGraphData(graphName, config); + @Nullable MapValue sourceNodePropertyValues = null; @Nullable MapValue targetNodePropertyValues = null; NodeLabelToken sourceNodeLabels = NodeLabelTokens.missing(); @@ -168,16 +168,6 @@ void projectNextRelationship( } } - var data = initGraphData( - graphName, - config, - targetNodePropertyValues, - sourceNodePropertyValues, - targetNodeLabels, - sourceNodeLabels, - relationshipConfig - ); - data.update( sourceNode, targetNode, @@ -189,15 +179,7 @@ void projectNextRelationship( ); } - private GraphImporter initGraphData( - TextValue graphName, - AnyValue config, - @Nullable MapValue sourceNodePropertyValues, - @Nullable MapValue targetNodePropertyValues, - NodeLabelToken sourceNodeLabels, - NodeLabelToken targetNodeLabels, - AnyValue relationshipConfig - ) { + private GraphImporter initGraphData(TextValue graphName, AnyValue config) { var data = this.importer; if (data != null) { return data; @@ -212,13 +194,7 @@ private GraphImporter initGraphData( this.username, this.databaseId, config, - sourceNodePropertyValues, - targetNodePropertyValues, - sourceNodeLabels, - targetNodeLabels, - relationshipConfig, - this.canWriteToDatabase, - this.lock + this.canWriteToDatabase ); } return data; @@ -334,11 +310,9 @@ private static final class GraphImporter { private final String graphName; private final GraphProjectFromCypherAggregationConfig config; private final LazyIdMapBuilder idMapBuilder; -// private final @Nullable List relationshipPropertySchemas; private final boolean canWriteToDatabase; private final ExtractNodeId extractNodeId; - private final Lock lock; private final Map relImporters; private final ImmutableGraphSchema.Builder graphSchemaBuilder; @@ -346,16 +320,12 @@ private GraphImporter( String graphName, GraphProjectFromCypherAggregationConfig config, LazyIdMapBuilder idMapBuilder, - @Nullable List relationshipPropertySchemas, - boolean canWriteToDatabase, - Lock lock + boolean canWriteToDatabase ) { this.graphName = graphName; this.config = config; this.idMapBuilder = idMapBuilder; -// this.relationshipPropertySchemas = relationshipPropertySchemas; this.canWriteToDatabase = canWriteToDatabase; - this.lock = lock; this.relImporters = new ConcurrentHashMap<>(); this.graphSchemaBuilder = ImmutableGraphSchema.builder(); this.extractNodeId = new ExtractNodeId(); @@ -366,13 +336,7 @@ static GraphImporter of( String username, DatabaseId databaseId, AnyValue configMap, - @Nullable MapValue sourceNodePropertyValues, - @Nullable MapValue targetNodePropertyValues, - NodeLabelToken sourceNodeLabels, - NodeLabelToken targetNodeLabels, - AnyValue relationshipConfig, - boolean canWriteToDatabase, - Lock lock + boolean canWriteToDatabase ) { var graphName = graphNameValue.stringValue(); @@ -384,24 +348,9 @@ static GraphImporter of( (configMap instanceof MapValue) ? (MapValue) configMap : MapValue.EMPTY ); - var idMapBuilder = idMapBuilder( - sourceNodeLabels, - sourceNodePropertyValues, - targetNodeLabels, - targetNodePropertyValues, - config.readConcurrency() - ); - - var relationshipPropertySchemas = relationshipPropertySchemas(relationshipConfig); + var idMapBuilder = idMapBuilder(config.readConcurrency()); - return new GraphImporter( - graphName, - config, - idMapBuilder, - relationshipPropertySchemas, - canWriteToDatabase, - lock - ); + return new GraphImporter(graphName, config, idMapBuilder, canWriteToDatabase); } private static void validateGraphName(String graphName, String username, DatabaseId databaseId) { @@ -410,46 +359,10 @@ private static void validateGraphName(String graphName, String username, Databas } } - private static LazyIdMapBuilder idMapBuilder( - NodeLabelToken sourceNodeLabels, - @Nullable MapValue sourceNodeProperties, - NodeLabelToken targetNodeLabels, - @Nullable MapValue targetNodeProperties, - int readConcurrency - ) { + private static LazyIdMapBuilder idMapBuilder(int readConcurrency) { return new LazyIdMapBuilder(readConcurrency, true, true); } - private static @Nullable List relationshipPropertySchemas(AnyValue relationshipConfigValue) { - if (!(relationshipConfigValue instanceof MapValue)) { - return null; - } - - //noinspection PatternVariableCanBeUsed - var relationshipConfig = (MapValue) relationshipConfigValue; - - var relationshipPropertySchemas = new ArrayList(); - - // We need to do this before extracting the `relationshipProperties`, because - // we remove the original entry from the map during converting; also we remove null keys - // so we could not create a schema entry for properties that are absent on the current relationship - var relationshipPropertyKeys = relationshipConfig.get("properties"); - if (relationshipPropertyKeys instanceof MapValue) { - for (var propertyKey : ((MapValue) relationshipPropertyKeys).keySet()) { - relationshipPropertySchemas.add(RelationshipPropertySchema.of( - propertyKey, - ValueType.DOUBLE - )); - } - } - - if (relationshipPropertySchemas.isEmpty()) { - return null; - } - - return relationshipPropertySchemas; - } - void update( AnyValue sourceNode, AnyValue targetNode, @@ -590,26 +503,6 @@ private long loadNode( ); } - private static double loadOneRelationshipProperty( - @NotNull MapValue relationshipProperties, - String relationshipPropertyKey - ) { - var propertyValue = relationshipProperties.get(relationshipPropertyKey); - return ReadHelper.extractValue(propertyValue, DefaultValue.DOUBLE_DEFAULT_FALLBACK); - } - - private static double[] loadMultipleRelationshipProperties( - @NotNull MapValue relationshipProperties, - List relationshipPropertyKeys - ) { - var propertyValues = new double[relationshipPropertyKeys.size()]; - Arrays.setAll(propertyValues, i -> { - var relationshipPropertyKey = relationshipPropertyKeys.get(i).key(); - return loadOneRelationshipProperty(relationshipProperties, relationshipPropertyKey); - }); - return propertyValues; - } - private AdjacencyCompressor.ValueMapper buildNodesWithProperties(GraphStoreBuilder graphStoreBuilder) { var idMapAndProperties = this.idMapBuilder.build(); @@ -648,14 +541,6 @@ private void buildRelationshipsWithProperties( ); var schema = relationships.relationshipSchema(relationshipType); relationshipSchemas.put(relationshipType, schema.get(relationshipType)); -// schema -// .entries() -// .forEach(entry -> relationshipSchemas.merge( -// relationshipType, -// entry, -// RelationshipSchemaEntry::union -// )); - relationshipImportResultBuilder.putImportResult(relationshipType, relationships); }); From 5f95c54ce2871cd4c8e5131eaabfa213d7a96eb0 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Mon, 23 Jan 2023 15:52:08 +0100 Subject: [PATCH 102/400] Remove and undo some more things that were added during bug fixing --- .../gds/core/loading/CSRGraphStoreUtil.java | 24 ------------------- .../gds/core/loading/LazyIdMapBuilder.java | 6 ----- .../loading/construction/GraphFactory.java | 7 +++--- .../neo4j/gds/projection/GraphAggregator.java | 1 - .../api/schema/RelationshipSchemaEntry.java | 2 +- 5 files changed, 4 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index aa1cd8c8b9f..c45d86ce69d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -32,11 +32,9 @@ import org.neo4j.gds.api.properties.graph.GraphPropertyStore; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.huge.HugeGraph; @@ -44,7 +42,6 @@ import java.util.Map; import java.util.Optional; -import java.util.function.Function; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -180,27 +177,6 @@ private static Optional constructRelationshipProperti } - public static void extractNodeProperties( - ImmutableNodes.Builder nodeImportResultBuilder, - Function nodeSchema, - Map nodeProperties - ) { - NodePropertyStore.Builder propertyStoreBuilder = NodePropertyStore.builder(); - nodeProperties.forEach((propertyKey, propertyValues) -> { - var propertySchema = nodeSchema.apply(propertyKey); - propertyStoreBuilder.putIfAbsent( - propertyKey, - NodeProperty.of( - propertyKey, - propertySchema.state(), - propertyValues, - propertySchema.defaultValue() - ) - ); - }); - nodeImportResultBuilder.properties(propertyStoreBuilder.build()); - } - // TODO: remove this method public static GraphSchema computeGraphSchema( Nodes nodes, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index 57d682136a4..651cd470436 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -23,7 +23,6 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodeLabelToken; @@ -31,7 +30,6 @@ import org.neo4j.gds.core.loading.construction.PropertyValues; import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; -import java.util.Map; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicBoolean; @@ -114,10 +112,6 @@ public interface HighLimitIdMapAndProperties { NodeSchema schema(); NodePropertyStore propertyStore(); - - default Map properties() { - return propertyStore().propertyValues(); - } } public HighLimitIdMapAndProperties build() { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 3f2e9e12c3d..b0da169f8b7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -222,10 +222,9 @@ static RelationshipsBuilder relationshipsBuilder( Optional validateRelationships, Optional concurrency, Optional indexInverse, - Optional executorService, - Optional loadRelationshipProperties + Optional executorService ) { - var doLoadRelationshipProperties = loadRelationshipProperties.orElse(!propertyConfigs.isEmpty()); + var loadRelationshipProperties = !propertyConfigs.isEmpty(); var aggregations = propertyConfigs.isEmpty() ? new Aggregation[]{aggregation.orElse(Aggregation.DEFAULT)} @@ -291,7 +290,7 @@ static RelationshipsBuilder relationshipsBuilder( .bufferSize(bufferSize) .propertyConfigs(propertyConfigs) .isMultiGraph(isMultiGraph) - .loadRelationshipProperty(doLoadRelationshipProperties) + .loadRelationshipProperty(loadRelationshipProperties) .direction(Direction.fromOrientation(actualOrientation)) .executorService(executorService.orElse(Pools.DEFAULT)) .concurrency(finalConcurrency); diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index 14af0205f00..2e154e30b58 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -518,7 +518,6 @@ private AdjacencyCompressor.ValueMapper buildNodesWithProperties(GraphStoreBuild .properties(idMapAndProperties.propertyStore()) .build(); - // graphStoreBuilder.nodes(nodesBuilder.schema(nodeSchema).build()); graphStoreBuilder.nodes(nodes); // Relationships are added using their intermediate node ids. diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java index 2599cd6547e..d468a96b722 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java @@ -65,7 +65,7 @@ boolean isUndirected() { } @Override - public RelationshipSchemaEntry union(RelationshipSchemaEntry other) { + RelationshipSchemaEntry union(RelationshipSchemaEntry other) { if (!other.identifier().equals(this.identifier())) { throw new UnsupportedOperationException( formatWithLocale( From 9bd3b22b10d0f39ab9e8b44b4ed9be009e28b027 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Mon, 23 Jan 2023 15:59:52 +0100 Subject: [PATCH 103/400] Remove unnecessary comments from test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Max Kießling --- .../java/org/neo4j/gds/projection/CypherAggregationTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index 3b885e744b7..aff73b20f28 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -182,13 +182,10 @@ void testDifferentPropertySchemas() { var graph = GraphStoreCatalog.get("", db.databaseName(), "g").graphStore().getUnion(); - // these fail to separate the property schemas assertThat(graph.schema().nodeSchema().get(NodeLabel.of("a")).properties().keySet()).isEmpty(); assertThat(graph.schema().nodeSchema().get(NodeLabel.of("b")).properties().keySet()).containsExactly("x"); assertThat(graph.schema().nodeSchema().get(NodeLabel.of("c")).properties().keySet()).containsExactly("y"); - // these also fail like that, but if one of them has no props (like rel does) no properties are retained at all - // so 2 problems to fix assertThat(graph .schema() .relationshipSchema() From cb44963542748ea53d10a5f2e09b104506174bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 19 Jan 2023 16:58:42 +0100 Subject: [PATCH 104/400] Replace DataClass in NodeProjection This also avoids accessing static base class members via derived types. --- etc/spotbugs/spotbugs-exclude.xml | 2 +- ...NodeProjection.java => NodeProjection.java} | 18 +++++++++--------- .../org/neo4j/gds/NodeProjectionsTest.java | 2 +- .../gds/catalog/GraphProjectProcTest.java | 2 +- .../java/org/neo4j/gds/AlgoBaseProcTest.java | 2 +- .../neo4j/gds/ConfigurableSeedConfigTest.java | 2 +- .../neo4j/gds/GraphProjectConfigSupport.java | 2 +- .../org/neo4j/gds/MutateNodePropertyTest.java | 4 ++-- .../src/main/java/org/neo4j/gds/GdsCypher.java | 10 +++++----- .../neo4j/gds/GraphProjectConfigBuilders.java | 2 +- 10 files changed, 23 insertions(+), 23 deletions(-) rename graph-projection-api/src/main/java/org/neo4j/gds/{AbstractNodeProjection.java => NodeProjection.java} (87%) diff --git a/etc/spotbugs/spotbugs-exclude.xml b/etc/spotbugs/spotbugs-exclude.xml index 24d2964bf79..706ee305eed 100644 --- a/etc/spotbugs/spotbugs-exclude.xml +++ b/etc/spotbugs/spotbugs-exclude.xml @@ -697,7 +697,7 @@ - + diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjection.java b/graph-projection-api/src/main/java/org/neo4j/gds/NodeProjection.java similarity index 87% rename from graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjection.java rename to graph-projection-api/src/main/java/org/neo4j/gds/NodeProjection.java index 0e2f2be99c7..a8803ef1f63 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjection.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/NodeProjection.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.jetbrains.annotations.Nullable; -import org.neo4j.gds.annotation.DataClass; +import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.core.ConfigKeyValidation; import org.neo4j.gds.utils.StringFormatting; @@ -29,10 +29,10 @@ import java.util.Map; import java.util.TreeMap; -@DataClass -public abstract class AbstractNodeProjection extends ElementProjection { +@ValueClass +public abstract class NodeProjection extends ElementProjection { - private static final NodeProjection ALL = of(PROJECT_ALL); + private static final NodeProjection ALL = fromString(PROJECT_ALL); public abstract String label(); @@ -50,7 +50,7 @@ public boolean projectAll() { public static final String LABEL_KEY = "label"; public static NodeProjection of(String label) { - return NodeProjection.of(label, PropertyMappings.of()); + return ImmutableNodeProjection.of(label, PropertyMappings.of()); } public static NodeProjection all() { @@ -85,7 +85,7 @@ public static NodeProjection fromString(@Nullable String label) { public static NodeProjection fromMap(Map map, NodeLabel nodeLabel) { validateConfigKeys(map); String label = String.valueOf(map.getOrDefault(LABEL_KEY, nodeLabel.name)); - return create(map, properties -> NodeProjection.of(label, properties)); + return create(map, properties -> ImmutableNodeProjection.of(label, properties)); } @Override @@ -102,9 +102,9 @@ void writeToObject(Map value) { public NodeProjection withAdditionalPropertyMappings(PropertyMappings mappings) { PropertyMappings newMappings = properties().mergeWith(mappings); if (newMappings == properties()) { - return (NodeProjection) this; + return this; } - return ((NodeProjection) this).withProperties(newMappings); + return ((ImmutableNodeProjection) this).withProperties(newMappings); } public static Builder builder() { @@ -116,7 +116,7 @@ private static void validateConfigKeys(Map map) { } @org.immutables.builder.Builder.AccessibleFields - public static final class Builder extends NodeProjection.Builder implements InlineProperties { + public static final class Builder extends ImmutableNodeProjection.Builder implements InlineProperties { private InlinePropertiesBuilder propertiesBuilder; diff --git a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java index 48b4a4ceb74..9ddc32f43ec 100644 --- a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java +++ b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java @@ -39,9 +39,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.text.MatchesPattern.matchesPattern; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.neo4j.gds.AbstractNodeProjection.LABEL_KEY; import static org.neo4j.gds.ElementProjection.PROPERTIES_KEY; import static org.neo4j.gds.NodeLabel.ALL_NODES; +import static org.neo4j.gds.NodeProjection.LABEL_KEY; class NodeProjectionsTest { diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java index c86804cd3ff..040467e68e9 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphProjectProcTest.java @@ -82,8 +82,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.neo4j.gds.AbstractNodeProjection.LABEL_KEY; import static org.neo4j.gds.ElementProjection.PROPERTIES_KEY; +import static org.neo4j.gds.NodeProjection.LABEL_KEY; import static org.neo4j.gds.RelationshipProjection.AGGREGATION_KEY; import static org.neo4j.gds.RelationshipProjection.INDEX_INVERSE_KEY; import static org.neo4j.gds.RelationshipProjection.ORIENTATION_KEY; diff --git a/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java b/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java index 7da7024faea..de08fd5e181 100644 --- a/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java @@ -323,7 +323,7 @@ default void testRunOnEmptyGraph() { loadedGraphName, NodeProjections.of( Map.of( - NodeLabel.of("X"), NodeProjection.of("X", PropertyMappings.of(propertyMappings)) + NodeLabel.of("X"), ImmutableNodeProjection.of("X", PropertyMappings.of(propertyMappings)) ) ), relationshipProjections() diff --git a/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java b/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java index 88df2c89c96..b81e71a705e 100644 --- a/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java @@ -85,7 +85,7 @@ default void testSeedPropertyValidation() { GraphProjectFromStoreConfig graphProjectConfig = ImmutableGraphProjectFromStoreConfig.of( "", graphName, - NodeProjections.single(NodeLabel.of("A"), NodeProjection.of("A", PropertyMappings.of(nodeProperties))), + NodeProjections.single(NodeLabel.of("A"), ImmutableNodeProjection.of("A", PropertyMappings.of(nodeProperties))), allRelationshipsProjection() ); diff --git a/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java b/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java index e7372764e80..ccb2275fcb0 100644 --- a/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java +++ b/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java @@ -60,7 +60,7 @@ default GraphProjectFromStoreConfig withNameAndRelationshipProjections( graphName, AbstractNodeProjections.create(singletonMap( ALL_NODES, - NodeProjection.of(PROJECT_ALL, PropertyMappings.of(propertyMappings)) + ImmutableNodeProjection.of(PROJECT_ALL, PropertyMappings.of(propertyMappings)) )), rels ); diff --git a/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java b/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java index a0a4777b810..fd7a92acd57 100644 --- a/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java @@ -60,11 +60,11 @@ default void testWriteBackGraphMutationOnFilteredGraph() { StoreLoaderBuilder storeLoaderBuilder = new StoreLoaderBuilder() .databaseService(graphDb()) .graphName(graphName) - .addNodeProjection(NodeProjection.of( + .addNodeProjection(ImmutableNodeProjection.of( "A", PropertyMappings.of(nodeProperties().stream().map(PropertyMapping::of).collect(Collectors.toList())) )) - .addNodeProjection(NodeProjection.of( + .addNodeProjection(ImmutableNodeProjection.of( "B", PropertyMappings.of(nodeProperties().stream().map(PropertyMapping::of).collect(Collectors.toList())) )); diff --git a/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java b/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java index b4425b02b2e..c90348046c8 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java +++ b/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java @@ -817,8 +817,8 @@ private static MinimalObject toMinimalObject( ElementProjection projection, ElementIdentifier identifier ) { - if (projection instanceof AbstractNodeProjection) { - return toMinimalObject(((AbstractNodeProjection) projection), identifier); + if (projection instanceof NodeProjection) { + return toMinimalObject(((NodeProjection) projection), identifier); } if (projection instanceof RelationshipProjection) { return toMinimalObject(((RelationshipProjection) projection), identifier); @@ -827,7 +827,7 @@ private static MinimalObject toMinimalObject( } private static MinimalObject toMinimalObject( - AbstractNodeProjection projection, + NodeProjection projection, ElementIdentifier identifier ) { MinimalObject properties = toMinimalObject(projection.properties(), false); @@ -836,12 +836,12 @@ private static MinimalObject toMinimalObject( } Map value = new LinkedHashMap<>(); - value.put(AbstractNodeProjection.LABEL_KEY, projection.label()); + value.put(NodeProjection.LABEL_KEY, projection.label()); properties.toObject().ifPresent(o -> value.put(PROPERTIES_KEY, o)); return MinimalObject.map(value); } - private static boolean matchesLabel(String label, AbstractNodeProjection projection) { + private static boolean matchesLabel(String label, NodeProjection projection) { return Objects.equals(projection.label(), label); } diff --git a/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java b/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java index 9055707b5cd..aa798440440 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java +++ b/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java @@ -70,7 +70,7 @@ public static GraphProjectFromStoreConfig storeConfig( ) { // Node projections Map tempNP = new LinkedHashMap<>(); - nodeLabels.forEach(label -> tempNP.put(label, NodeProjection.of(label, PropertyMappings.of()))); + nodeLabels.forEach(label -> tempNP.put(label, NodeProjection.of(label))); nodeProjections.forEach(np -> tempNP.put(np.label(), np)); nodeProjectionsWithIdentifier.forEach(tempNP::put); From 0753a4013e1e3581eee75604561f3f9176438a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 19 Jan 2023 17:10:48 +0100 Subject: [PATCH 105/400] Replace DataClass in NodeProjections This also avoids accessing static base class members via derived types. Also makes the code more aligned to other immutable classes in our code base. --- .../gds/config/GraphProjectFromStoreConfig.java | 4 ++-- .../gds/config/RandomGraphGeneratorConfig.java | 5 +++-- ...NodeProjections.java => NodeProjections.java} | 16 ++++++++-------- .../java/org/neo4j/gds/AlgoBaseProcTest.java | 2 +- .../org/neo4j/gds/GraphProjectConfigSupport.java | 2 +- .../neo4j/gds/GraphProjectConfigBuilders.java | 2 +- 6 files changed, 16 insertions(+), 15 deletions(-) rename graph-projection-api/src/main/java/org/neo4j/gds/{AbstractNodeProjections.java => NodeProjections.java} (94%) diff --git a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java index ca75eeab5d2..89c7979680a 100644 --- a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java @@ -52,8 +52,8 @@ public interface GraphProjectFromStoreConfig extends GraphProjectConfig { String RELATIONSHIP_PROPERTIES_KEY = "relationshipProperties"; @Key(NODE_PROJECTION_KEY) - @ConvertWith(method = "org.neo4j.gds.AbstractNodeProjections#fromObject") - @Configuration.ToMapValue("org.neo4j.gds.AbstractNodeProjections#toObject") + @ConvertWith(method = "org.neo4j.gds.NodeProjections#fromObject") + @Configuration.ToMapValue("org.neo4j.gds.NodeProjections#toObject") NodeProjections nodeProjections(); @Key(RELATIONSHIP_PROJECTION_KEY) diff --git a/core/src/main/java/org/neo4j/gds/config/RandomGraphGeneratorConfig.java b/core/src/main/java/org/neo4j/gds/config/RandomGraphGeneratorConfig.java index a6bd7c8582f..3f4e211c731 100644 --- a/core/src/main/java/org/neo4j/gds/config/RandomGraphGeneratorConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/RandomGraphGeneratorConfig.java @@ -21,6 +21,7 @@ import org.immutables.value.Value; import org.jetbrains.annotations.Nullable; +import org.neo4j.gds.ImmutableNodeProjections; import org.neo4j.gds.ImmutableRelationshipProjections; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; @@ -97,9 +98,9 @@ default Map relationshipProperty() { } @Value.Default - @Configuration.ToMapValue("org.neo4j.gds.AbstractNodeProjections#toObject") + @Configuration.ToMapValue("org.neo4j.gds.NodeProjections#toObject") default NodeProjections nodeProjections() { - return NodeProjections.builder() + return ImmutableNodeProjections.builder() .putProjection( NodeLabel.of(nodeCount() + "_Nodes"), NodeProjection.of(nodeCount() + "_Nodes")) diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java b/graph-projection-api/src/main/java/org/neo4j/gds/NodeProjections.java similarity index 94% rename from graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java rename to graph-projection-api/src/main/java/org/neo4j/gds/NodeProjections.java index bc585203be6..be2fb2d0b2c 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractNodeProjections.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/NodeProjections.java @@ -21,7 +21,7 @@ import org.immutables.value.Value; import org.jetbrains.annotations.Nullable; -import org.neo4j.gds.annotation.DataClass; +import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.utils.StringFormatting; import java.util.HashMap; @@ -37,9 +37,9 @@ import static org.neo4j.gds.NodeLabel.ALL_NODES; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -@DataClass +@ValueClass @Value.Immutable(singleton = true) -public abstract class AbstractNodeProjections extends AbstractProjections { +public abstract class NodeProjections extends AbstractProjections { public static final NodeProjections ALL = create(singletonMap(ALL_NODES, NodeProjection.all())); @@ -112,11 +112,11 @@ public static NodeProjections create(Map projections) "An empty node projection was given; at least one node label must be projected." ); } - return NodeProjections.of(unmodifiableMap(projections)); + return ImmutableNodeProjections.of(unmodifiableMap(projections)); } public static NodeProjections single(NodeLabel label, NodeProjection projection) { - return NodeProjections.of(Map.of(label, projection)); + return ImmutableNodeProjections.of(Map.of(label, projection)); } public static NodeProjections all() { @@ -125,7 +125,7 @@ public static NodeProjections all() { public NodeProjections addPropertyMappings(PropertyMappings mappings) { if (!mappings.hasMappings()) { - return NodeProjections.copyOf(this); + return ImmutableNodeProjections.copyOf(this); } Map newProjections = projections().entrySet().stream().collect(toMap( Map.Entry::getKey, @@ -150,7 +150,7 @@ public String labelProjection() { } public boolean isEmpty() { - return this == NodeProjections.of(); + return this == ImmutableNodeProjections.of(); } public Map toObject() { @@ -161,7 +161,7 @@ public Map toObject() { return value; } - public static Map toObject(AbstractNodeProjections nodeProjections) { + public static Map toObject(NodeProjections nodeProjections) { return nodeProjections.toObject(); } diff --git a/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java b/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java index de08fd5e181..9381fd2b281 100644 --- a/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java @@ -321,7 +321,7 @@ default void testRunOnEmptyGraph() { GraphProjectConfig graphProjectConfig = withNameAndProjections( "", loadedGraphName, - NodeProjections.of( + ImmutableNodeProjections.of( Map.of( NodeLabel.of("X"), ImmutableNodeProjection.of("X", PropertyMappings.of(propertyMappings)) ) diff --git a/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java b/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java index ccb2275fcb0..669c790e3a0 100644 --- a/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java +++ b/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java @@ -58,7 +58,7 @@ default GraphProjectFromStoreConfig withNameAndRelationshipProjections( return ImmutableGraphProjectFromStoreConfig.of( userName, graphName, - AbstractNodeProjections.create(singletonMap( + NodeProjections.create(singletonMap( ALL_NODES, ImmutableNodeProjection.of(PROJECT_ALL, PropertyMappings.of(propertyMappings)) )), diff --git a/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java b/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java index aa798440440..37c8638788b 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java +++ b/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java @@ -111,7 +111,7 @@ public static GraphProjectFromStoreConfig storeConfig( .withDefaultAggregation(aggregation) .build(); - NodeProjections np = NodeProjections.of(tempNP.entrySet().stream().collect(Collectors.toMap( + NodeProjections np = ImmutableNodeProjections.of(tempNP.entrySet().stream().collect(Collectors.toMap( e -> NodeLabel.of(e.getKey()), Map.Entry::getValue ))); From e9dc7c7916796ee8f0d14022473b32059045086d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 19 Jan 2023 17:33:40 +0100 Subject: [PATCH 106/400] Replace DataClass in PropertyMappings This removes the last usage of the DataClass --- .../scaling/ScalePropertiesBaseConfig.java | 2 +- .../org/neo4j/gds/annotation/DataClass.java | 46 ------------------- .../org.immutables.value.immutable | 1 - .../config/GraphProjectFromStoreConfig.java | 8 ++-- .../loading/CypherRelationshipLoader.java | 3 +- .../java/org/neo4j/gds/ElementProjection.java | 8 ++-- ...rtyMappings.java => PropertyMappings.java} | 28 +++++------ .../core/io/GraphStoreExporterBaseConfig.java | 4 +- .../java/org/neo4j/gds/AlgoBaseProcTest.java | 2 +- .../neo4j/gds/ConfigurableSeedConfigTest.java | 5 +- .../neo4j/gds/GraphProjectConfigSupport.java | 2 +- .../org/neo4j/gds/MutateNodePropertyTest.java | 5 +- .../org/neo4j/gds/MutatePropertyProcTest.java | 2 +- .../main/java/org/neo4j/gds/GdsCypher.java | 6 +-- .../neo4j/gds/GraphProjectConfigBuilders.java | 2 +- 15 files changed, 37 insertions(+), 87 deletions(-) delete mode 100644 annotations/src/main/java/org/neo4j/gds/annotation/DataClass.java rename graph-projection-api/src/main/java/org/neo4j/gds/{AbstractPropertyMappings.java => PropertyMappings.java} (87%) diff --git a/algo-common/src/main/java/org/neo4j/gds/scaling/ScalePropertiesBaseConfig.java b/algo-common/src/main/java/org/neo4j/gds/scaling/ScalePropertiesBaseConfig.java index 7c74b650b7f..9bfafb065b8 100644 --- a/algo-common/src/main/java/org/neo4j/gds/scaling/ScalePropertiesBaseConfig.java +++ b/algo-common/src/main/java/org/neo4j/gds/scaling/ScalePropertiesBaseConfig.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.stream.Collectors; -import static org.neo4j.gds.AbstractPropertyMappings.fromObject; +import static org.neo4j.gds.PropertyMappings.fromObject; @ValueClass @SuppressWarnings("immutables:subtype") diff --git a/annotations/src/main/java/org/neo4j/gds/annotation/DataClass.java b/annotations/src/main/java/org/neo4j/gds/annotation/DataClass.java deleted file mode 100644 index 513dbe9689b..00000000000 --- a/annotations/src/main/java/org/neo4j/gds/annotation/DataClass.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.annotation; - -import org.immutables.value.Value; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target({ElementType.PACKAGE, ElementType.TYPE}) -@Retention(RetentionPolicy.CLASS) -@Value.Style( - allParameters = true, - builderVisibility = Value.Style.BuilderVisibility.SAME, - clearBuilder = true, - deepImmutablesDetection = true, - deferCollectionAllocation = true, - depluralize = true, - headerComments = true, - jdkOnly = true, - optionalAcceptNullable = true, - typeAbstract = "Abstract*", - typeImmutable = "*", - visibility = Value.Style.ImplementationVisibility.SAME -) -public @interface DataClass { -} diff --git a/annotations/src/main/resources/META-INF/annotations/org.immutables.value.immutable b/annotations/src/main/resources/META-INF/annotations/org.immutables.value.immutable index b067608b47a..cf5adfc05a9 100644 --- a/annotations/src/main/resources/META-INF/annotations/org.immutables.value.immutable +++ b/annotations/src/main/resources/META-INF/annotations/org.immutables.value.immutable @@ -1,2 +1 @@ -org.neo4j.gds.annotation.DataClass org.neo4j.gds.annotation.ValueClass diff --git a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java index 89c7979680a..1b001c86356 100644 --- a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromStoreConfig.java @@ -63,16 +63,16 @@ public interface GraphProjectFromStoreConfig extends GraphProjectConfig { @Value.Default @Value.Parameter(false) - @Configuration.ConvertWith(method = "org.neo4j.gds.AbstractPropertyMappings#fromObject") - @Configuration.ToMapValue("org.neo4j.gds.AbstractPropertyMappings#toObject") + @Configuration.ConvertWith(method = "org.neo4j.gds.PropertyMappings#fromObject") + @Configuration.ToMapValue("org.neo4j.gds.PropertyMappings#toObject") default PropertyMappings nodeProperties() { return PropertyMappings.of(); } @Value.Default @Value.Parameter(false) - @Configuration.ConvertWith(method = "org.neo4j.gds.AbstractPropertyMappings#fromObject") - @Configuration.ToMapValue("org.neo4j.gds.AbstractPropertyMappings#toObject") + @Configuration.ConvertWith(method = "org.neo4j.gds.PropertyMappings#fromObject") + @Configuration.ToMapValue("org.neo4j.gds.PropertyMappings#toObject") default PropertyMappings relationshipProperties() { return PropertyMappings.of(); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java index 432cbd0d587..8e9479fe40a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java @@ -23,6 +23,7 @@ import org.eclipse.collections.impl.map.mutable.primitive.ObjectDoubleHashMap; import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap; import org.immutables.value.Value; +import org.neo4j.gds.ImmutablePropertyMappings; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; @@ -123,7 +124,7 @@ BatchLoadResult loadSingleBatch(InternalTransaction tx, int bufferSize) { )) .collect(Collectors.toList()); - initFromPropertyMappings(PropertyMappings.of(propertyMappings)); + initFromPropertyMappings(ImmutablePropertyMappings.of(propertyMappings)); initializedFromResult = true; } diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/ElementProjection.java b/graph-projection-api/src/main/java/org/neo4j/gds/ElementProjection.java index eb2ca77f4e9..33cbc328406 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/ElementProjection.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/ElementProjection.java @@ -102,7 +102,7 @@ default Self addProperty( ) { inlineBuilder() .propertiesBuilder() - .addMapping(propertyKey, neoPropertyKey, defaultValue, aggregation); + .addMapping(PropertyMapping.of(propertyKey, neoPropertyKey, defaultValue, aggregation)); return (Self) this; } @@ -126,7 +126,7 @@ default void buildProperties() { static final class InlinePropertiesBuilder { private final Supplier getProperties; private final Consumer setProperties; - private AbstractPropertyMappings.Builder propertiesBuilder; + private PropertyMappings.Builder propertiesBuilder; InlinePropertiesBuilder( Supplier getProperties, @@ -151,9 +151,9 @@ private void build() { } } - private AbstractPropertyMappings.Builder propertiesBuilder() { + private PropertyMappings.Builder propertiesBuilder() { if (propertiesBuilder == null) { - propertiesBuilder = AbstractPropertyMappings.builder(); + propertiesBuilder = PropertyMappings.builder(); PropertyMappings properties = getProperties.get(); if (properties != null) { propertiesBuilder.from(properties); diff --git a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractPropertyMappings.java b/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMappings.java similarity index 87% rename from graph-projection-api/src/main/java/org/neo4j/gds/AbstractPropertyMappings.java rename to graph-projection-api/src/main/java/org/neo4j/gds/PropertyMappings.java index 3f900bff305..9fbb7e3fd2a 100644 --- a/graph-projection-api/src/main/java/org/neo4j/gds/AbstractPropertyMappings.java +++ b/graph-projection-api/src/main/java/org/neo4j/gds/PropertyMappings.java @@ -19,12 +19,10 @@ */ package org.neo4j.gds; -import org.eclipse.collections.api.tuple.primitive.IntObjectPair; import org.immutables.builder.Builder.AccessibleFields; import org.immutables.value.Value; -import org.neo4j.gds.annotation.DataClass; +import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.core.utils.CollectionUtil; import java.util.Arrays; import java.util.Iterator; @@ -40,17 +38,17 @@ import static java.util.Collections.singletonMap; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; -@DataClass +@ValueClass @Value.Immutable(singleton = true) -public abstract class AbstractPropertyMappings implements Iterable { +public abstract class PropertyMappings implements Iterable { public abstract List mappings(); public static PropertyMappings of(PropertyMapping... mappings) { if (mappings == null) { - return PropertyMappings.of(); + return ImmutablePropertyMappings.of(); } - return PropertyMappings.of(Arrays.asList(mappings)); + return ImmutablePropertyMappings.of(Arrays.asList(mappings)); } public static PropertyMappings fromObject(Object relPropertyMapping) { @@ -58,9 +56,9 @@ public static PropertyMappings fromObject(Object relPropertyMapping) { } public static PropertyMappings fromObject(Object relPropertyMapping, Aggregation defaultAggregation) { - if (relPropertyMapping instanceof PropertyMappings) { - PropertyMappings properties = (PropertyMappings) relPropertyMapping; - return PropertyMappings.builder().from(properties).withDefaultAggregation(defaultAggregation).build(); + if (relPropertyMapping instanceof ImmutablePropertyMappings) { + ImmutablePropertyMappings properties = (ImmutablePropertyMappings) relPropertyMapping; + return ImmutablePropertyMappings.builder().from(properties).withDefaultAggregation(defaultAggregation).build(); } if (relPropertyMapping instanceof String) { String propertyMapping = (String) relPropertyMapping; @@ -95,7 +93,7 @@ public static PropertyMappings fromObject(Object relPropertyMapping, Aggregation } } - public static Map toObject(AbstractPropertyMappings propertyMappings) { + public static Map toObject(PropertyMappings propertyMappings) { return propertyMappings.toObject(true); } @@ -112,10 +110,6 @@ public Iterator iterator() { return mappings().iterator(); } - public Stream> enumerate() { - return CollectionUtil.enumerate(mappings()); - } - public boolean hasMappings() { return !mappings().isEmpty(); } @@ -144,7 +138,7 @@ public PropertyMappings mergeWith(PropertyMappings other) { return other; } if (!other.hasMappings()) { - return PropertyMappings.copyOf(this); + return ImmutablePropertyMappings.copyOf(this); } Builder builder = PropertyMappings.builder(); builder.addMappings(Stream.concat(stream(), other.stream()).distinct()); @@ -168,7 +162,7 @@ public static Builder builder() { } @AccessibleFields - public static final class Builder extends PropertyMappings.Builder { + public static final class Builder extends ImmutablePropertyMappings.Builder { private Aggregation aggregation; diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporterBaseConfig.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporterBaseConfig.java index 6c83af8280a..3025e5d2a46 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporterBaseConfig.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreExporterBaseConfig.java @@ -51,8 +51,8 @@ default int batchSize() { @Value.Default @Value.Parameter(false) - @Configuration.ConvertWith(method = "org.neo4j.gds.AbstractPropertyMappings#fromObject") - @Configuration.ToMapValue("org.neo4j.gds.AbstractPropertyMappings#toObject") + @Configuration.ConvertWith(method = "org.neo4j.gds.PropertyMappings#fromObject") + @Configuration.ToMapValue("org.neo4j.gds.PropertyMappings#toObject") default org.neo4j.gds.PropertyMappings additionalNodeProperties() { return org.neo4j.gds.PropertyMappings.of(); } diff --git a/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java b/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java index 9381fd2b281..2d03f397403 100644 --- a/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/AlgoBaseProcTest.java @@ -323,7 +323,7 @@ default void testRunOnEmptyGraph() { loadedGraphName, ImmutableNodeProjections.of( Map.of( - NodeLabel.of("X"), ImmutableNodeProjection.of("X", PropertyMappings.of(propertyMappings)) + NodeLabel.of("X"), ImmutableNodeProjection.of("X", ImmutablePropertyMappings.of(propertyMappings)) ) ), relationshipProjections() diff --git a/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java b/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java index b81e71a705e..bb5c1618216 100644 --- a/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/ConfigurableSeedConfigTest.java @@ -85,7 +85,10 @@ default void testSeedPropertyValidation() { GraphProjectFromStoreConfig graphProjectConfig = ImmutableGraphProjectFromStoreConfig.of( "", graphName, - NodeProjections.single(NodeLabel.of("A"), ImmutableNodeProjection.of("A", PropertyMappings.of(nodeProperties))), + NodeProjections.single( + NodeLabel.of("A"), + ImmutableNodeProjection.of("A", ImmutablePropertyMappings.of(nodeProperties)) + ), allRelationshipsProjection() ); diff --git a/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java b/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java index 669c790e3a0..20e908d23e5 100644 --- a/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java +++ b/proc/test/src/main/java/org/neo4j/gds/GraphProjectConfigSupport.java @@ -60,7 +60,7 @@ default GraphProjectFromStoreConfig withNameAndRelationshipProjections( graphName, NodeProjections.create(singletonMap( ALL_NODES, - ImmutableNodeProjection.of(PROJECT_ALL, PropertyMappings.of(propertyMappings)) + ImmutableNodeProjection.of(PROJECT_ALL, ImmutablePropertyMappings.of(propertyMappings)) )), rels ); diff --git a/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java b/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java index fd7a92acd57..a1d9d3cd952 100644 --- a/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/MutateNodePropertyTest.java @@ -30,7 +30,6 @@ import java.lang.reflect.InvocationTargetException; import java.util.Collections; import java.util.Map; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -62,11 +61,11 @@ default void testWriteBackGraphMutationOnFilteredGraph() { .graphName(graphName) .addNodeProjection(ImmutableNodeProjection.of( "A", - PropertyMappings.of(nodeProperties().stream().map(PropertyMapping::of).collect(Collectors.toList())) + PropertyMappings.fromObject(nodeProperties()) )) .addNodeProjection(ImmutableNodeProjection.of( "B", - PropertyMappings.of(nodeProperties().stream().map(PropertyMapping::of).collect(Collectors.toList())) + PropertyMappings.fromObject(nodeProperties()) )); diff --git a/proc/test/src/main/java/org/neo4j/gds/MutatePropertyProcTest.java b/proc/test/src/main/java/org/neo4j/gds/MutatePropertyProcTest.java index 849b188652d..8a97f2dec50 100644 --- a/proc/test/src/main/java/org/neo4j/gds/MutatePropertyProcTest.java +++ b/proc/test/src/main/java/org/neo4j/gds/MutatePropertyProcTest.java @@ -103,7 +103,7 @@ default void testGraphMutationOnFilteredGraph() { .orElse(Orientation.NATURAL); GraphStore graphStore = new TestNativeGraphLoader(graphDb()) .withLabels("A", "B") - .withNodeProperties(PropertyMappings.of(nodeProperties() + .withNodeProperties(ImmutablePropertyMappings.of(nodeProperties() .stream() .map(PropertyMapping::of) .collect(Collectors.toList()))) diff --git a/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java b/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java index c90348046c8..f6f40029712 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java +++ b/test-utils/src/main/java/org/neo4j/gds/GdsCypher.java @@ -662,8 +662,8 @@ static GraphProjectFromStoreConfig inlineGraphProjectConfig( .graphName(graphName.orElse("")) .nodeProjections(NodeProjections.create(nodeProjections)) .relationshipProjections(ImmutableRelationshipProjections.builder().putAllProjections(relProjections).build()) - .nodeProperties(PropertyMappings.of(nodeProperties)) - .relationshipProperties(PropertyMappings.of(relProperties)) + .nodeProperties(ImmutablePropertyMappings.of(nodeProperties)) + .relationshipProperties(ImmutablePropertyMappings.of(relProperties)) .build(); } @@ -881,7 +881,7 @@ private static boolean matchesType(String type, RelationshipProjection projectio } private static MinimalObject toMinimalObject( - AbstractPropertyMappings propertyMappings, + PropertyMappings propertyMappings, boolean includeAggregation ) { List mappings = propertyMappings.mappings(); diff --git a/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java b/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java index 37c8638788b..3ee1d7eb854 100644 --- a/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java +++ b/test-utils/src/main/java/org/neo4j/gds/GraphProjectConfigBuilders.java @@ -126,7 +126,7 @@ public static GraphProjectFromStoreConfig storeConfig( .graphName(graphName.orElse("")) .nodeProjections(np) .relationshipProjections(rp) - .nodeProperties(PropertyMappings.of(nodeProperties)) + .nodeProperties(ImmutablePropertyMappings.of(nodeProperties)) .relationshipProperties(relationshipPropertyMappings) .readConcurrency(concurrency.orElse(ConcurrencyConfig.DEFAULT_CONCURRENCY)) .jobId(jobId.orElse(new JobId())) From 86b16c10d797d124ed9a33e99dcf2b2c0a264ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 20 Jan 2023 09:22:44 +0100 Subject: [PATCH 107/400] Fix test usages --- .../schema/GraphSchemaIntegrationTest.java | 12 ++-- .../GraphProjectFromCypherConfigTest.java | 6 +- .../GraphProjectFromStoreConfigTest.java | 66 ++++++++++++------- ...aderMultipleRelTypesAndPropertiesTest.java | 45 ++++--------- .../gds/core/loading/GraphStoreTest.java | 30 ++++----- .../storageengine/InMemoryNodeCursorTest.java | 5 +- .../InMemoryStorageEngineTest.java | 5 +- .../org/neo4j/gds/NodeProjectionsTest.java | 45 ++++++------- .../ModularityOptimizationMutateProcTest.java | 18 +++-- .../GraphDropNodePropertiesProcTest.java | 21 +++--- .../GraphStreamNodePropertiesProcTest.java | 24 +++---- .../GraphWriteNodePropertiesProcTest.java | 23 +++---- .../graphsage/GraphSageBaseProcTest.java | 43 +++++------- .../graphsage/GraphSageTrainProcTest.java | 19 +++--- .../gds/GraphProjectConfigBuildersTest.java | 26 ++------ 15 files changed, 173 insertions(+), 215 deletions(-) diff --git a/core/src/test/java/org/neo4j/gds/api/schema/GraphSchemaIntegrationTest.java b/core/src/test/java/org/neo4j/gds/api/schema/GraphSchemaIntegrationTest.java index aac7189122b..e5e25f6ef32 100644 --- a/core/src/test/java/org/neo4j/gds/api/schema/GraphSchemaIntegrationTest.java +++ b/core/src/test/java/org/neo4j/gds/api/schema/GraphSchemaIntegrationTest.java @@ -27,7 +27,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.StoreLoaderBuilder; @@ -37,7 +36,6 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.core.Aggregation; -import java.util.List; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -62,10 +60,10 @@ void computesCorrectNodeSchema(PropertySchema expectedSchema, PropertyMapping pr Graph graph = new StoreLoaderBuilder() .databaseService(db) .addNodeProjection( - NodeProjection.of( - "Node", - PropertyMappings.of(List.of(propertyMapping)) - ) + NodeProjection.builder() + .label("Node") + .addProperty(propertyMapping) + .build() ) .build() .graph(); @@ -81,7 +79,7 @@ void computesCorrectRelationshipSchema(RelationshipPropertySchema expectedSchema .addRelationshipProjection( RelationshipProjection.builder() .type("REL") - .properties(PropertyMappings.of(List.of(propertyMapping))) + .addProperties(propertyMapping) .build() ) .build() diff --git a/core/src/test/java/org/neo4j/gds/config/GraphProjectFromCypherConfigTest.java b/core/src/test/java/org/neo4j/gds/config/GraphProjectFromCypherConfigTest.java index 6a6bdb062a4..bc013a0044f 100644 --- a/core/src/test/java/org/neo4j/gds/config/GraphProjectFromCypherConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/GraphProjectFromCypherConfigTest.java @@ -24,8 +24,8 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.AbstractProjections; +import org.neo4j.gds.ImmutableNodeProjections; import org.neo4j.gds.ImmutableRelationshipProjections; -import org.neo4j.gds.NodeProjections; import org.neo4j.gds.core.CypherMapWrapper; import java.util.Map; @@ -71,9 +71,9 @@ void omitCypherParametersFromToMap() { static Stream invalidKeys() { return Stream.of( - Arguments.of(GraphProjectFromStoreConfig.NODE_PROJECTION_KEY, NodeProjections.of()), + Arguments.of(GraphProjectFromStoreConfig.NODE_PROJECTION_KEY, ImmutableNodeProjections.of()), Arguments.of(GraphProjectFromStoreConfig.RELATIONSHIP_PROJECTION_KEY, ImmutableRelationshipProjections.of()), - Arguments.of(GraphProjectFromStoreConfig.NODE_PROPERTIES_KEY, NodeProjections.of()) + Arguments.of(GraphProjectFromStoreConfig.NODE_PROPERTIES_KEY, ImmutableNodeProjections.of()) ); } diff --git a/core/src/test/java/org/neo4j/gds/config/GraphProjectFromStoreConfigTest.java b/core/src/test/java/org/neo4j/gds/config/GraphProjectFromStoreConfigTest.java index a1b14db1a80..443148ee0f6 100644 --- a/core/src/test/java/org/neo4j/gds/config/GraphProjectFromStoreConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/GraphProjectFromStoreConfigTest.java @@ -25,6 +25,7 @@ import org.neo4j.gds.NodeProjection; import org.neo4j.gds.NodeProjections; import org.neo4j.gds.Orientation; +import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipProjections; @@ -32,7 +33,6 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.core.Aggregation; -import java.util.Collections; import java.util.Set; import static org.hamcrest.MatcherAssert.assertThat; @@ -46,14 +46,18 @@ class GraphProjectFromStoreConfigTest { @Test void testThrowOnOverlappingNodeProperties() { - PropertyMappings propertyMappings = PropertyMappings.builder() - .addMapping("duplicate", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); - - NodeProjections nodeProjections = NodeProjections.create(Collections.singletonMap( - NodeLabel.of("A"), NodeProjection.of("A", propertyMappings) + PropertyMappings propertyMappings = PropertyMappings.of(PropertyMapping.of( + "duplicate", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE )); + var nodeProjections = NodeProjections.single( + NodeLabel.of("A"), + NodeProjection.builder().label("A").properties(propertyMappings).build() + ); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> ImmutableGraphProjectFromStoreConfig.builder() .graphName("graph") @@ -68,9 +72,12 @@ void testThrowOnOverlappingNodeProperties() { @Test void testThrowOnOverlappingRelProperties() { - PropertyMappings propertyMappings = PropertyMappings.builder() - .addMapping("duplicate", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); + var propertyMappings = PropertyMappings.of(PropertyMapping.of( + "duplicate", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE + )); RelationshipProjections relProjections = ImmutableRelationshipProjections.single( RelationshipType.of("A"), @@ -95,18 +102,26 @@ void testThrowOnOverlappingRelProperties() { @Test void testMergingOfNodePropertiesAndProjections() { - PropertyMappings propertyMappings1 = PropertyMappings.builder() - .addMapping("foo", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); - - PropertyMappings propertyMappings2 = PropertyMappings.builder() - .addMapping("bar", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); + var propertyMappings1 = PropertyMappings.of(PropertyMapping.of( + "foo", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE + ) + ); - NodeProjections nodeProjections = NodeProjections.create(Collections.singletonMap( - NodeLabel.of("A"), NodeProjection.of("A", propertyMappings2) + var propertyMappings2 = PropertyMappings.of(PropertyMapping.of( + "bar", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE )); + var nodeProjections = NodeProjections.single( + NodeLabel.of("A"), + NodeProjection.builder().label("A").properties(propertyMappings2).build() + ); + GraphProjectFromStoreConfig graphProjectConfig = ImmutableGraphProjectFromStoreConfig.builder() .graphName("graph") .relationshipProjections(RelationshipProjections.ALL) @@ -122,13 +137,14 @@ void testMergingOfNodePropertiesAndProjections() { @Test void testMergingOfRelationshipPropertiesAndProjections() { - PropertyMappings propertyMappings1 = PropertyMappings.builder() - .addMapping("foo", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); + var propertyMappings1 = PropertyMappings.of(PropertyMapping.of("foo", "foo", DefaultValue.of(0.0), Aggregation.NONE)); - PropertyMappings propertyMappings2 = PropertyMappings.builder() - .addMapping("bar", "foo", DefaultValue.of(0.0), Aggregation.NONE) - .build(); + var propertyMappings2 = PropertyMappings.of(PropertyMapping.of( + "bar", + "foo", + DefaultValue.of(0.0), + Aggregation.NONE + )); RelationshipProjections relProjections = ImmutableRelationshipProjections.single( RelationshipType.of("A"), diff --git a/core/src/test/java/org/neo4j/gds/core/GraphLoaderMultipleRelTypesAndPropertiesTest.java b/core/src/test/java/org/neo4j/gds/core/GraphLoaderMultipleRelTypesAndPropertiesTest.java index 302452f96ce..c623c4e6656 100644 --- a/core/src/test/java/org/neo4j/gds/core/GraphLoaderMultipleRelTypesAndPropertiesTest.java +++ b/core/src/test/java/org/neo4j/gds/core/GraphLoaderMultipleRelTypesAndPropertiesTest.java @@ -29,7 +29,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.StoreLoaderBuilder; @@ -89,25 +88,14 @@ void setup() { @Test void nodeProjectionsWithExclusiveProperties() { GraphStore graphStore = new StoreLoaderBuilder() - .putNodeProjectionsWithIdentifier( - "N1", - NodeProjection.of( - "Node1", - PropertyMappings.builder().addMapping(PropertyMapping.of("prop1", 0.0D)).build() - ) - ).putNodeProjectionsWithIdentifier( - "N2", - NodeProjection.of( - "Node1", - PropertyMappings.of() - ) - ).putNodeProjectionsWithIdentifier( - "N3", - NodeProjection.of( - "Node2", - PropertyMappings.builder().addMapping(PropertyMapping.of("prop2", 1.0D)).build() - ) - ).graphName("myGraph") + .putNodeProjectionsWithIdentifier("N1", + NodeProjection.builder().label("Node1").addProperty(PropertyMapping.of("prop1", 0.0D)).build() + ) + .putNodeProjectionsWithIdentifier("N2", NodeProjection.of("Node1")) + .putNodeProjectionsWithIdentifier("N3", + NodeProjection.builder().label("Node2").addProperty(PropertyMapping.of("prop2", 1.0D)).build() + ) + .graphName("myGraph") .databaseService(db) .build() .graphStore(); @@ -134,19 +122,14 @@ void nodeProjectionsWithAndWithoutLabel() { GraphStore graphStore = new StoreLoaderBuilder() .putNodeProjectionsWithIdentifier( allIdentifier.name(), - NodeProjection.of( - "*", - PropertyMappings.builder() - .addMapping(PropertyMapping.of("prop1", 42.0D)) - .addMapping(PropertyMapping.of("prop2", 8.0D)) - .build() - ) + NodeProjection + .builder() + .label("*") + .addProperties(PropertyMapping.of("prop1", 42.0D), PropertyMapping.of("prop2", 8.0D)) + .build() ).putNodeProjectionsWithIdentifier( node2Identifier.name(), - NodeProjection.of( - "Node2", - PropertyMappings.builder().addMapping(PropertyMapping.of("prop2", 8.0D)).build() - ) + NodeProjection.builder().label("Node2").addProperty(PropertyMapping.of("prop2", 8.0D)).build() ).graphName("myGraph") .databaseService(db) .build() diff --git a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java index c5c46c427d2..eae8ef0a836 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java @@ -279,18 +279,18 @@ void nodeOnlyGraph() { private static List nodeProjections() { NodeProjection aMapping = NodeProjection.builder() .label("A") - .properties(PropertyMappings.of(Arrays.asList( + .addProperties( PropertyMapping.of("nodeProperty", -1D), PropertyMapping.of("a", -1D) - ))) + ) .build(); NodeProjection bMapping = NodeProjection.builder() .label("B") - .properties(PropertyMappings.of(Arrays.asList( + .addProperties( PropertyMapping.of("nodeProperty", -1D), PropertyMapping.of("b", -1D) - ))) + ) .build(); return Arrays.asList(aMapping, bMapping); @@ -302,32 +302,24 @@ private static List relationshipProjections() { .type("T1") .orientation(Orientation.NATURAL) .aggregation(Aggregation.NONE) - .properties( - PropertyMappings.builder() - .addMapping("property1", "property1", DefaultValue.of(42D), Aggregation.NONE) - .addMapping("property2", "property2", DefaultValue.of(1337D), Aggregation.NONE) - .build() + .addProperties( + PropertyMapping.of("property1", "property1", DefaultValue.of(42D), Aggregation.NONE), + PropertyMapping.of("property2", "property2", DefaultValue.of(1337D), Aggregation.NONE) ).build(); RelationshipProjection t2Mapping = RelationshipProjection.builder() .type("T2") .orientation(Orientation.NATURAL) .aggregation(Aggregation.NONE) - .properties( - PropertyMappings.builder() - .addMapping("property1", "property1", DefaultValue.of(42D), Aggregation.NONE) - .build() - ).build(); + .addProperty(PropertyMapping.of("property1", "property1", DefaultValue.of(42D), Aggregation.NONE)) + .build(); RelationshipProjection t3Mapping = RelationshipProjection.builder() .type("T3") .orientation(Orientation.NATURAL) .aggregation(Aggregation.NONE) - .properties( - PropertyMappings.builder() - .addMapping("property2", "property2", DefaultValue.of(42D), Aggregation.NONE) - .build() - ).build(); + .addProperty(PropertyMapping.of("property2", "property2", DefaultValue.of(42D), Aggregation.NONE)) + .build(); return Arrays.asList(t1Mapping, t2Mapping, t3Mapping); } diff --git a/cypher/cypher-test/src/test/java/org/neo4j/gds/storageengine/InMemoryNodeCursorTest.java b/cypher/cypher-test/src/test/java/org/neo4j/gds/storageengine/InMemoryNodeCursorTest.java index d00f8d603ca..70ea3a26862 100644 --- a/cypher/cypher-test/src/test/java/org/neo4j/gds/storageengine/InMemoryNodeCursorTest.java +++ b/cypher/cypher-test/src/test/java/org/neo4j/gds/storageengine/InMemoryNodeCursorTest.java @@ -22,7 +22,6 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; @@ -53,8 +52,8 @@ protected void onSetup() { protected GraphStore graphStore() { return new StoreLoaderBuilder() .databaseService(db) - .addNodeProjection(NodeProjection.of("A", PropertyMappings.of(PropertyMapping.of("prop1")))) - .addNodeProjection(NodeProjection.of("B", PropertyMappings.of(PropertyMapping.of("prop2")))) + .addNodeProjection(NodeProjection.builder().label("A").addProperty(PropertyMapping.of("prop1")).build()) + .addNodeProjection(NodeProjection.builder().label("B").addProperty(PropertyMapping.of("prop2")).build()) .build() .graphStore(); } diff --git a/cypher/cypher-test/src/test/java/org/neo4j/internal/recordstorage/InMemoryStorageEngineTest.java b/cypher/cypher-test/src/test/java/org/neo4j/internal/recordstorage/InMemoryStorageEngineTest.java index 336f9a77309..dba53ca9cf3 100644 --- a/cypher/cypher-test/src/test/java/org/neo4j/internal/recordstorage/InMemoryStorageEngineTest.java +++ b/cypher/cypher-test/src/test/java/org/neo4j/internal/recordstorage/InMemoryStorageEngineTest.java @@ -23,7 +23,6 @@ import org.neo4j.gds.NodeProjection; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.api.GraphStore; @@ -45,8 +44,8 @@ class InMemoryStorageEngineTest extends CypherTest { protected GraphStore graphStore() { return new StoreLoaderBuilder() .databaseService(db) - .addNodeProjection(NodeProjection.of("A", PropertyMappings.of(PropertyMapping.of("prop1")))) - .addNodeProjection(NodeProjection.of("B", PropertyMappings.of(PropertyMapping.of("prop2")))) + .addNodeProjection(NodeProjection.builder().label("A").addProperty(PropertyMapping.of("prop1")).build()) + .addNodeProjection(NodeProjection.builder().label("B").addProperty(PropertyMapping.of("prop2")).build()) .addRelationshipProjection(RelationshipProjection.of("REL", Orientation.NATURAL)) .build() .graphStore(); diff --git a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java index 9ddc32f43ec..424357c7ac1 100644 --- a/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java +++ b/graph-projection-api/src/test/java/org/neo4j/gds/NodeProjectionsTest.java @@ -33,7 +33,6 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -50,10 +49,10 @@ class NodeProjectionsTest { void syntacticSugars(Object argument) { NodeProjections actual = NodeProjections.fromObject(argument); - NodeProjections expected = NodeProjections.builder().projections(singletonMap( + NodeProjections expected = NodeProjections.single( NodeLabel.of("A"), NodeProjection.builder().label("A").properties(PropertyMappings.of()).build() - )).build(); + ); assertThat( actual, @@ -73,19 +72,17 @@ void shouldParseWithProperties() { ) )); - NodeProjections expected = NodeProjections.builder().projections(singletonMap( + NodeProjections expected = NodeProjections.single( NodeLabel.of("MY_LABEL"), NodeProjection .builder() .label("A") - .properties(PropertyMappings - .builder() - .addMapping(PropertyMapping.of("prop1", DefaultValue.DEFAULT)) - .addMapping(PropertyMapping.of("prop2", DefaultValue.DEFAULT)) - .build() + .addProperties( + PropertyMapping.of("prop1", DefaultValue.DEFAULT), + PropertyMapping.of("prop2", DefaultValue.DEFAULT) ) .build() - )).build(); + ); assertThat( actual, @@ -98,10 +95,10 @@ void shouldParseWithProperties() { void shouldParseMultipleLabels() { NodeProjections actual = NodeProjections.fromObject(Arrays.asList("A", "B")); - NodeProjections expected = NodeProjections.builder() - .putProjection(NodeLabel.of("A"), NodeProjection.builder().label("A").build()) - .putProjection(NodeLabel.of("B"), NodeProjection.builder().label("B").build()) - .build(); + NodeProjections expected = NodeProjections.create(Map.of( + NodeLabel.of("A"), NodeProjection.builder().label("A").build(), + NodeLabel.of("B"), NodeProjection.builder().label("B").build() + )); assertThat(actual, equalTo(expected)); assertThat(actual.labelProjection(), equalTo("A, B")); @@ -111,14 +108,14 @@ void shouldParseMultipleLabels() { void shouldSupportStar() { NodeProjections actual = NodeProjections.fromObject("*"); - NodeProjections expected = NodeProjections.builder().projections(singletonMap( + NodeProjections expected = NodeProjections.single( ALL_NODES, NodeProjection .builder() .label("*") .properties(PropertyMappings.of()) .build() - )).build(); + ); assertThat( actual, @@ -190,21 +187,19 @@ void shouldFailOnAmbiguousNodePropertyDefinition( PropertyMapping second, String... messages ) { - var builder = NodeProjections.builder().projections(Map.of( + var builder = ImmutableNodeProjections.builder().projections(Map.of( NodeLabel.of("A"), NodeProjection .builder() .label("A") .addProperty(first) .build(), - NodeLabel.of("B"), NodeProjection .builder() .label("B") .addProperty(second) .build() - )); var throwableAssert = assertThatThrownBy(builder::build) @@ -225,19 +220,17 @@ void shouldSupportCaseInsensitiveConfigKeys() { ) )); - NodeProjections expected = NodeProjections.builder().projections(singletonMap( + NodeProjections expected = NodeProjections.single( NodeLabel.of("MY_LABEL"), NodeProjection .builder() .label("A") - .properties(PropertyMappings - .builder() - .addMapping(PropertyMapping.of("prop1", DefaultValue.DEFAULT)) - .addMapping(PropertyMapping.of("prop2", DefaultValue.DEFAULT)) - .build() + .addProperties( + PropertyMapping.of("prop1", DefaultValue.DEFAULT), + PropertyMapping.of("prop2", DefaultValue.DEFAULT) ) .build() - )).build(); + ); assertThat( actual, diff --git a/proc/beta/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java b/proc/beta/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java index 0cdfbbeec4b..6d390e2dc1a 100644 --- a/proc/beta/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java +++ b/proc/beta/src/test/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationMutateProcTest.java @@ -23,23 +23,22 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.AlgoBaseProc; import org.neo4j.gds.GdsCypher; -import org.neo4j.gds.ImmutableRelationshipProjections; import org.neo4j.gds.MutateNodePropertyTest; import org.neo4j.gds.NodeProjections; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; +import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.config.GraphProjectFromStoreConfig; -import org.neo4j.gds.config.ImmutableGraphProjectFromStoreConfig; +import org.neo4j.gds.config.GraphProjectFromStoreConfigImpl; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.graphdb.GraphDatabaseService; import java.util.Arrays; -import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -206,21 +205,20 @@ GdsCypher.ModeBuildStage explicitAlgoBuildStage() { } static String graphProjectQuery() { - GraphProjectFromStoreConfig config = ImmutableGraphProjectFromStoreConfig + GraphProjectFromStoreConfig config = GraphProjectFromStoreConfigImpl .builder() .graphName("") - .nodeProjections(NodeProjections.of()) + .username("") + .nodeProjections(NodeProjections.all()) .nodeProperties(PropertyMappings.fromObject(Arrays.asList("seed1", "seed2"))) - .relationshipProjections(ImmutableRelationshipProjections.builder() - .putProjection( + .relationshipProjections(RelationshipProjections.single( ALL_RELATIONSHIPS, RelationshipProjection.builder() .type("TYPE") .orientation(Orientation.UNDIRECTED) - .properties(PropertyMappings.of( - Collections.singletonList(PropertyMapping.of("weight", 1D)))) + .addProperty(PropertyMapping.of("weight", 1D)) .build() - ).build() + ) ).build(); return GdsCypher diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProcTest.java index 6ad4639490b..753ae9c0bf3 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProcTest.java @@ -27,7 +27,6 @@ import org.neo4j.gds.GdsCypher; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.core.loading.CatalogRequest; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.utils.warnings.GlobalUserLogStore; @@ -90,19 +89,19 @@ void setup() throws Exception { runQuery(GdsCypher.call(TEST_GRAPH_DIFFERENT_PROPERTIES) .graphProject() - .withNodeLabel("A", NodeProjection.of( - "A", - PropertyMappings.of().withMappings( + .withNodeLabel("A", NodeProjection.builder() + .label("A") + .addProperties( PropertyMapping.of("nodeProp1", 1337), PropertyMapping.of("nodeProp2", 1337) - ) - )) - .withNodeLabel("B", NodeProjection.of( - "B", - PropertyMappings.of().withMappings( + ).build() + ) + .withNodeLabel("B", NodeProjection.builder() + .label("B") + .addProperty( PropertyMapping.of("nodeProp1", 1337) - ) - )) + ).build() + ) .withAnyRelationshipType() .yields() ); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java index 733869a368b..4c86e3d06ac 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamNodePropertiesProcTest.java @@ -31,7 +31,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; @@ -100,19 +99,20 @@ void setup() throws Exception { runQuery(GdsCypher.call(TEST_GRAPH_DIFFERENT_PROPERTIES) .graphProject() - .withNodeLabel("A", NodeProjection.of( - "A", - PropertyMappings.of().withMappings( + .withNodeLabel("A", NodeProjection.builder() + .label("A") + .addProperties( PropertyMapping.of("newNodeProp1", "nodeProp1", 1337), PropertyMapping.of("newNodeProp2", "nodeProp2", 1337) - ) - )) - .withNodeLabel("B", NodeProjection.of( - "B", - PropertyMappings.of().withMappings( - PropertyMapping.of("newNodeProp1", "nodeProp1", 1337) - ) - )) + ).build() + ) + .withNodeLabel("B", + NodeProjection + .builder() + .label("B") + .addProperty(PropertyMapping.of("newNodeProp1", "nodeProp1", 1337)) + .build() + ) .withAnyRelationshipType() .yields() ); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProcTest.java index d4b20a0b5fa..95b2ab869c8 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodePropertiesProcTest.java @@ -31,7 +31,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; @@ -109,19 +108,21 @@ void setup() throws Exception { runQuery(GdsCypher.call(TEST_GRAPH_DIFFERENT_PROPERTIES) .graphProject() - .withNodeLabel("A", NodeProjection.of( - "A", - PropertyMappings.of().withMappings( + .withNodeLabel("A", NodeProjection.builder() + .label("A") + .addProperties( PropertyMapping.of("newNodeProp1", "nodeProp1", 1337), PropertyMapping.of("newNodeProp2", "nodeProp2", 1337) - ) - )) - .withNodeLabel("B", NodeProjection.of( + ).build() + ) + .withNodeLabel( "B", - PropertyMappings.of().withMappings( - PropertyMapping.of("newNodeProp1", "nodeProp1", 1337) - ) - )) + NodeProjection + .builder() + .label("B") + .addProperty(PropertyMapping.of("newNodeProp1", "nodeProp1", 1337)) + .build() + ) .withAnyRelationshipType() .yields() ); diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageBaseProcTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageBaseProcTest.java index 125525117ca..f4d91beef45 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageBaseProcTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageBaseProcTest.java @@ -29,7 +29,6 @@ import org.neo4j.gds.NodeProjections; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.catalog.GraphProjectProc; @@ -165,18 +164,14 @@ static Stream missingNodeProperties() { Arguments.of( ImmutableGraphProjectFromStoreConfig.builder() .graphName("implicitWeightedGraph") - .nodeProjections(NodeProjections - .builder() - .putProjection( - NodeLabel.of("King"), - NodeProjection.of( - "King", - PropertyMappings.of( - PropertyMapping.of("age") - ) - ) - ) - .build()) + .nodeProjections(NodeProjections.single( + NodeLabel.of("King"), + NodeProjection.builder() + .label("King") + .addProperty( + PropertyMapping.of("age") + ).build() + )) .relationshipProjections(RelationshipProjections.fromString("REL") ).build(), List.of("birth_year", "death_year"), @@ -186,19 +181,15 @@ static Stream missingNodeProperties() { Arguments.of( ImmutableGraphProjectFromStoreConfig.builder() .graphName("implicitWeightedGraph") - .nodeProjections(NodeProjections - .builder() - .putProjection( - NodeLabel.of("King"), - NodeProjection.of( - "King", - PropertyMappings.of( - PropertyMapping.of("age"), - PropertyMapping.of("birth_year") - ) - ) - ) - .build()) + .nodeProjections(NodeProjections.single( + NodeLabel.of("King"), + NodeProjection.builder() + .label("King") + .addProperties( + PropertyMapping.of("age"), + PropertyMapping.of("birth_year") + ).build() + )) .relationshipProjections(RelationshipProjections.fromString("REL") ).build(), List.of("death_year"), diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java index 2ba78761f65..f4251641d00 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java @@ -25,7 +25,6 @@ import org.neo4j.gds.GdsCypher; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.PropertyMapping; -import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.TestProcedureRunner; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.schema.GraphSchema; @@ -101,16 +100,18 @@ void runsTrainingOnMultiLabelGraph() { .graphProject() .withNodeLabel( "A", - NodeProjection.of("A", PropertyMappings.of( - PropertyMapping.of("a1"), - PropertyMapping.of("a2") - )) + NodeProjection + .builder() + .label("A") + .addProperties(PropertyMapping.of("a1"), PropertyMapping.of("a2")) + .build() ).withNodeLabel( "B", - NodeProjection.of("B", PropertyMappings.of( - PropertyMapping.of("b1"), - PropertyMapping.of("b2") - )) + NodeProjection + .builder() + .label("B") + .addProperties(PropertyMapping.of("b1"), PropertyMapping.of("b2")) + .build() ) .withAnyRelationshipType() .yields(); diff --git a/test-utils/src/test/java/org/neo4j/gds/GraphProjectConfigBuildersTest.java b/test-utils/src/test/java/org/neo4j/gds/GraphProjectConfigBuildersTest.java index 98d918ddfda..d9cd1e4377c 100644 --- a/test-utils/src/test/java/org/neo4j/gds/GraphProjectConfigBuildersTest.java +++ b/test-utils/src/test/java/org/neo4j/gds/GraphProjectConfigBuildersTest.java @@ -58,9 +58,7 @@ static Stream storeConfigs() { Arguments.arguments( new StoreConfigBuilder().jobId(jobId).build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(ALL_NODES, NodeProjection.all()) - .build()) + .nodeProjections(NodeProjections.single(ALL_NODES, NodeProjection.all())) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection(ALL_RELATIONSHIPS, RelationshipProjection.ALL) .build()) @@ -72,9 +70,7 @@ static Stream storeConfigs() { Arguments.arguments( new StoreConfigBuilder().addNodeLabel("Foo").addRelationshipType("BAR").jobId(jobId).build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(NodeLabel.of("Foo"), NodeProjection.of("Foo", PropertyMappings.of())) - .build()) + .nodeProjections(NodeProjections.single(NodeLabel.of("Foo"), NodeProjection.of("Foo"))) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), @@ -93,9 +89,7 @@ static Stream storeConfigs() { .jobId(jobId) .build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(NodeLabel.of("Foo"), NodeProjection.of("Foo", PropertyMappings.of())) - .build()) + .nodeProjections(NodeProjections.single(NodeLabel.of("Foo"), NodeProjection.of("Foo"))) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), @@ -115,9 +109,7 @@ static Stream storeConfigs() { .jobId(jobId) .build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(NodeLabel.of("Foo"), NodeProjection.of("Foo", PropertyMappings.of())) - .build()) + .nodeProjections(NodeProjections.single(NodeLabel.of("Foo"), NodeProjection.of("Foo"))) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), @@ -138,9 +130,7 @@ static Stream storeConfigs() { .jobId(jobId) .build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection(NodeLabel.of("Foo"), NodeProjection.of("Foo", PropertyMappings.of())) - .build()) + .nodeProjections(NodeProjections.single(NodeLabel.of("Foo"), NodeProjection.of("Foo"))) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), @@ -168,15 +158,13 @@ static Stream storeConfigs() { .jobId(jobId) .build(), ImmutableGraphProjectFromStoreConfig.builder().username("").graphName("") - .nodeProjections(NodeProjections.builder() - .putProjection( + .nodeProjections(NodeProjections.single( NodeLabel.of("Foo"), NodeProjection.builder() .label("Foo") .addProperty(PropertyMapping.of("nProp", DefaultValue.of(23.0D))) .build() - ) - .build()) + )) .relationshipProjections(ImmutableRelationshipProjections.builder() .putProjection( RelationshipType.of("BAR"), From 1de7ec2c3d3c2c87fefca4f76605dd9ed65df8f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 26 Jan 2023 08:55:09 +0100 Subject: [PATCH 108/400] Publish 5.4 compat --- build.gradle | 4 + .../5.4/neo4j-kernel-adapter/build.gradle | 63 ++ .../gds/compat/_54/Neo4jProxyFactoryImpl.java | 44 + .../compat/_54/SettingProxyFactoryImpl.java | 44 + .../compat/_54/BoltTransactionRunnerImpl.java | 96 ++ .../gds/compat/_54/CallableProcedureImpl.java | 53 + .../gds/compat/_54/CompatAccessModeImpl.java | 44 + .../_54/CompatGraphDatabaseAPIImpl.java | 44 + .../gds/compat/_54/CompatIndexQueryImpl.java | 31 + .../_54/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_54/CompositeNodeCursorImpl.java | 32 + .../gds/compat/_54/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 ++ .../gds/compat/_54/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_54/Neo4jProxyImpl.java | 915 ++++++++++++++++++ .../compat/_54/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_54/PartitionedStoreScan.java | 58 ++ .../_54/ReferencePropertyReference.java | 49 + .../compat/_54/ScanBasedStoreScanImpl.java | 40 + .../compat/_54/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_54/SettingProxyImpl.java | 82 ++ .../org/neo4j/gds/compat/_54/TestLogImpl.java | 148 +++ .../compat/_54/VirtualRelationshipImpl.java | 41 + .../5.4/storage-engine-adapter/build.gradle | 65 ++ .../_54/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 106 ++ .../compat/_54/InMemoryCountsStoreImpl.java | 110 +++ .../_54/InMemoryMetaDataProviderImpl.java | 190 ++++ .../gds/compat/_54/InMemoryNodeCursor.java | 82 ++ .../_54/InMemoryNodePropertyCursor.java | 45 + .../compat/_54/InMemoryPropertyCursor.java | 71 ++ .../_54/InMemoryPropertySelectionImpl.java | 55 ++ .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_54/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_54/InMemoryStorageEngineFactory.java | 563 +++++++++++ .../compat/_54/InMemoryStorageEngineImpl.java | 287 ++++++ .../compat/_54/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_54/InMemoryStoreVersion.java | 68 ++ .../_54/InMemoryTransactionIdStoreImpl.java | 75 ++ .../gds/compat/_54/InMemoryVersionCheck.java | 61 ++ .../_54/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_54/StorageEngineProxyImpl.java | 156 +++ .../InMemoryLogVersionRepository.java | 71 ++ .../InMemoryStorageCommandReaderFactory.java | 43 + .../InMemoryStorageReader54.java | 331 +++++++ gradle/dependencies.gradle | 1 + .../org/neo4j/gds/compat/Neo4jVersion.java | 7 +- .../neo4j/gds/compat/Neo4jVersionTest.java | 3 +- .../java/org/neo4j/gds/SysInfoProcTest.java | 13 + settings.gradle | 12 + 51 files changed, 4838 insertions(+), 2 deletions(-) create mode 100644 compatibility/5.4/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableProcedureImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatAccessModeImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatIndexQueryImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/PartitionedStoreScan.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ReferencePropertyReference.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/TestLogImpl.java create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/VirtualRelationshipImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/build.gradle create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java create mode 100644 cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java diff --git a/build.gradle b/build.gradle index c767e083418..fb98ff8da7a 100644 --- a/build.gradle +++ b/build.gradle @@ -31,11 +31,15 @@ ext { project(':neo4j-kernel-adapter-4.4'), project(':neo4j-kernel-adapter-5.1'), project(':neo4j-kernel-adapter-5.2'), + project(':neo4j-kernel-adapter-5.3'), + project(':neo4j-kernel-adapter-5.4'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), project(':storage-engine-adapter-5.1'), project(':storage-engine-adapter-5.2'), + project(':storage-engine-adapter-5.3'), + project(':storage-engine-adapter-5.4'), ] ] } diff --git a/compatibility/5.4/neo4j-kernel-adapter/build.gradle b/compatibility/5.4/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..f73ed67ab7f --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.4' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.4' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.4' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.4' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.4' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.4' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.4' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..1e38d77de3c --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.4 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.4 (placeholder)"; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..ff39f69c757 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.4 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.4 (placeholder)"; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/BoltTransactionRunnerImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..aa4c808530f --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/BoltTransactionRunnerImpl.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.bookmark.Bookmark; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.transaction.result.AdaptingBoltQuerySubscriber; +import org.neo4j.bolt.protocol.v41.message.request.RoutingContext; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new AdaptingBoltQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSucceeded(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.queryStatistics(); + } + + @Override + public AdaptingBoltQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + AdaptingBoltQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()) + ); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableProcedureImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableProcedureImpl.java new file mode 100644 index 00000000000..c73a48027a0 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatAccessModeImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatAccessModeImpl.java new file mode 100644 index 00000000000..06dfd97bd2d --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatGraphDatabaseAPIImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..06cedb1d578 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatIndexQueryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..adcd4483a1d --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatUsernameAuthSubjectImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..974b2cea35d --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompositeNodeCursorImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..7fcaba83573 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseLayoutImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..9f655fe2412 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..fb39c02a57e --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..3fe03256e7b --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_4; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.4"; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java new file mode 100644 index 00000000000..7e97c5697e1 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -0,0 +1,915 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Group; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public long pageCacheMemory() { + return pageCacheMemory.orElseGet(Configuration.super::pageCacheMemory); + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = new Group(0, null, null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor() { + var globalGroup = new Group(0, null, null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/NodeLabelIndexLookupImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..bedfd628c17 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/PartitionedStoreScan.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/PartitionedStoreScan.java new file mode 100644 index 00000000000..6d80192595d --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ReferencePropertyReference.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ReferencePropertyReference.java new file mode 100644 index 00000000000..1ccf92014fa --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ScanBasedStoreScanImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..2e84d278532 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..55d9d22773a --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_4; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.4"; + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java new file mode 100644 index 00000000000..d21ef5f3eb8 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/TestLogImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/TestLogImpl.java new file mode 100644 index 00000000000..6e75890ee50 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/TestLogImpl.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} + + diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/VirtualRelationshipImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..3d0f8a96dbb --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/cypher/5.4/storage-engine-adapter/build.gradle b/cypher/5.4/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..0feaddfc02f --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.4' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.4' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.4' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.4' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.4' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.4' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..e997b83e3da --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.4"; + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..d6825dc4902 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.LATEST; + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..f3f3546f7c3 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..282d71ea4fb --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + byteOffset, + logVersion + ); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } + + TransactionIdStore transactionIdStore() { + return this.transactionIdStore; + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java new file mode 100644 index 00000000000..5cc9cc6e713 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..f956b5a6d53 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..6420fa80c4c --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..6cc18b723c0 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..0d3efb743c1 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..9473f7cbb49 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..9abb9283d79 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..6b66f4e3153 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -0,0 +1,563 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-54"; + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + + public static final byte ID = 42; + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + cursorContextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker databaseReadOnlyChecker, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore(LogTailMetadata logTailMetadata) { + return new InMemoryMetaDataProviderImpl().transactionIdStore(); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository(LogTailMetadata logTailMetadata) { + return new InMemoryLogVersionRepository(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier throwingSupplier, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..159449bbf2b --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java @@ -0,0 +1,287 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader54; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private final StorageEngineIndexingBehaviour indexingBehaviour = () -> false; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return indexingBehaviour; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader54(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..b0f496aeda2 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java new file mode 100644 index 00000000000..b5663131022 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..40f845337e4 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4] + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + } + + @Override + protected CursorContext getEmptyCursorContext() { + return CursorContext.NULL_CONTEXT; + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java new file mode 100644 index 00000000000..45feb84ad9a --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._54.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..7f07bf1e11a --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_4; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.4"; + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..f99cb355377 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader54; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader54(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java new file mode 100644 index 00000000000..b491fbc7f32 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository() { + this(0,0); + } + + private InMemoryLogVersionRepository(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java new file mode 100644 index 00000000000..620a8aac11c --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java new file mode 100644 index 00000000000..d5ddf2ed531 --- /dev/null +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java @@ -0,0 +1,331 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._54.InMemoryNodeCursor; +import org.neo4j.gds.compat._54.InMemoryPropertyCursor; +import org.neo4j.gds.compat._54.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._54.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader54 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader54( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .properties() + .values() + .stream() + .mapToInt(map -> map.keySet().size()) + .sum(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .properties() + .values() + .stream() + .mapToInt(map -> map.keySet().size()) + .sum(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().properties().keySet().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 0d8e1206ea0..e4bbe89bde5 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -4,6 +4,7 @@ ext { '5.1' : properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2' : properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3' : properties.getOrDefault('neo4jVersion53', '5.3.0'), + '5.4' : properties.getOrDefault('neo4jVersion53', '5.4.0'), ] neo4jDefault = neos.'4.4' diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index c3f12691015..c8550611b9d 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -31,6 +31,7 @@ public enum Neo4jVersion { V_5_1, V_5_2, V_5_3, + V_5_4, V_RC; @Override @@ -44,8 +45,10 @@ public String toString() { return "5.2"; case V_5_3: return "5.3"; - case V_RC: + case V_5_4: return "5.4"; + case V_RC: + return "rc"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } @@ -119,6 +122,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 3) { return Neo4jVersion.V_5_3; } else if (minorVersion == 4) { + return Neo4jVersion.V_5_4; + } else if (minorVersion == 5) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 4368f4991b4..bab22d96e7f 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -44,7 +44,8 @@ class Neo4jVersionTest { "5.2.0-dev, V_5_2", "5.2.0, V_5_2", "5.3.0, V_5_3", - "5.4.0, V_RC", + "5.4.0, V_5_4", + "5.5.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index a53e7dc68c7..675e3a2a279 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -66,6 +66,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.3", "Neo4j Settings 5.3 (placeholder)", + "Neo4j 5.4", + "Neo4j 5.4 (placeholder)", + "Neo4j Settings 5.4", + "Neo4j Settings 5.4 (placeholder)", + "Neo4j DEV", "Neo4j DEV (placeholder)", "Neo4j Settings DEV", @@ -150,6 +155,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.3" ); break; + case V_5_4: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.4 (placeholder)", + "Neo4j Settings 5.4", + "Neo4j 5.4 (placeholder)", + "Neo4j 5.4" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC (placeholder)", diff --git a/settings.gradle b/settings.gradle index ac0c7d7ea73..dfb58392ff8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -130,6 +130,12 @@ project(':neo4j-kernel-adapter-5.1').projectDir = file('compatibility/5.1/neo4j- include('neo4j-kernel-adapter-5.2') project(':neo4j-kernel-adapter-5.2').projectDir = file('compatibility/5.2/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.3') +project(':neo4j-kernel-adapter-5.3').projectDir = file('compatibility/5.3/neo4j-kernel-adapter') + +include('neo4j-kernel-adapter-5.4') +project(':neo4j-kernel-adapter-5.4').projectDir = file('compatibility/5.4/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -208,6 +214,12 @@ project(':storage-engine-adapter-5.1').projectDir = file('cypher/5.1/storage-eng include('storage-engine-adapter-5.2') project(':storage-engine-adapter-5.2').projectDir = file('cypher/5.2/storage-engine-adapter') +include('storage-engine-adapter-5.3') +project(':storage-engine-adapter-5.3').projectDir = file('cypher/5.3/storage-engine-adapter') + +include('storage-engine-adapter-5.4') +project(':storage-engine-adapter-5.4').projectDir = file('cypher/5.4/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('cypher/api/storage-engine-adapter') From cb0d7c512a34c8bef4ec2501e946a2a564229d00 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 18 Jan 2023 10:37:46 +0100 Subject: [PATCH 109/400] Move creating new tx context to compat layer This change is required after neo-technology/neo4j#17457 --- .../neo4j/gds/compat/_44/Neo4jProxyImpl.java | 14 ++++++++++- .../neo4j/gds/compat/_51/Neo4jProxyImpl.java | 14 ++++++++++- .../neo4j/gds/compat/_52/Neo4jProxyImpl.java | 14 ++++++++++- .../neo4j/gds/compat/_53/Neo4jProxyImpl.java | 13 +++++++++++ .../org/neo4j/gds/compat/Neo4jProxyApi.java | 10 ++++++++ .../java/org/neo4j/gds/compat/Neo4jProxy.java | 12 ++++++++++ .../gds/core/loading/CypherRecordLoader.java | 23 ++++++++++++++++++- .../gds/compat/GraphDatabaseApiProxy.java | 21 ----------------- 8 files changed, 96 insertions(+), 25 deletions(-) diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index 8ef9e8ffcbb..b8d2acca3c4 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -108,7 +108,10 @@ import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; @@ -124,6 +127,7 @@ import org.neo4j.storageengine.api.PropertySelection; import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.ValueGroup; +import org.neo4j.values.virtual.MapValue; import java.io.IOException; import java.nio.file.Path; @@ -785,5 +789,13 @@ public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, Curso idGenerator.nextConsecutiveIdRange(size, false, cursorContext); } - + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } } diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index 5d085755c17..5709766e4d3 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -116,7 +116,10 @@ import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -134,6 +137,7 @@ import org.neo4j.util.Bits; import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; import java.io.IOException; import java.nio.file.Path; @@ -908,5 +912,13 @@ public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, Curso idGenerator.nextConsecutiveIdRange(size, false, cursorContext); } - + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } } diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index 2d151a763bd..a0d71680974 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -116,7 +116,10 @@ import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -134,6 +137,7 @@ import org.neo4j.util.Bits; import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; import java.io.IOException; import java.nio.file.Path; @@ -906,5 +910,13 @@ public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, Curso idGenerator.nextConsecutiveIdRange(size, false, cursorContext); } - + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } } diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index 49579510ce0..c1051e45511 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -116,7 +116,10 @@ import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormatSelector; import org.neo4j.kernel.impl.store.format.RecordFormats; @@ -134,6 +137,7 @@ import org.neo4j.util.Bits; import org.neo4j.values.storable.ValueCategory; import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; import java.io.IOException; import java.nio.file.Path; @@ -907,4 +911,13 @@ public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, Curso idGenerator.nextConsecutiveIdRange(size, false, cursorContext); } + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters); + } } diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index 6843ffc68d4..ccd863bacae 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -72,6 +72,9 @@ import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; @@ -80,6 +83,7 @@ import org.neo4j.procedure.Mode; import org.neo4j.scheduler.JobScheduler; import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.values.virtual.MapValue; import java.nio.file.Path; import java.util.List; @@ -309,4 +313,10 @@ UserFunctionSignature userFunctionSignature( void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext); + TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ); } diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index 92a01719d11..ea25d7a6205 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -72,6 +72,9 @@ import org.neo4j.kernel.api.procedure.CallableProcedure; import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; @@ -80,6 +83,7 @@ import org.neo4j.procedure.Mode; import org.neo4j.scheduler.JobScheduler; import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.values.virtual.MapValue; import java.nio.file.Path; import java.util.List; @@ -466,6 +470,14 @@ public static long transactionId(KernelTransactionHandle kernelTransactionHandle public static void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { IMPL.reserveNeo4jIds(generatorFactory, size, cursorContext); } + public static TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return IMPL.newQueryContext(contextFactory, tx, queryText, queryParameters); + } private Neo4jProxy() { throw new UnsupportedOperationException("No instances"); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java index 9f64d8a2282..636bbd9a7e3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRecordLoader.java @@ -21,24 +21,27 @@ import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.utils.StringJoining; import org.neo4j.graphdb.security.AuthorizationViolationException; import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.query.QueryExecution; import org.neo4j.kernel.impl.query.QueryExecutionEngine; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.util.ValueUtils; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.Locale; +import java.util.Map; import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.neo4j.gds.compat.GraphDatabaseApiProxy.runQueryWithoutClosingTheResult; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; abstract class CypherRecordLoader { @@ -146,4 +149,22 @@ private void validateMandatoryColumns(Collection allColumns) { )); } } + + private static QueryExecution runQueryWithoutClosingTheResult( + InternalTransaction tx, + String query, + Map params, + TransactionalContextFactory contextFactory, + QueryExecutionEngine executionEngine, + QuerySubscriber subscriber + ) { + var convertedParams = ValueUtils.asMapValue(params); + var context = Neo4jProxy.newQueryContext(contextFactory, tx, query, convertedParams); + try { + return executionEngine.executeQuery(query, convertedParams, context, false, subscriber); + } catch (QueryExecutionKernelException e) { + throw e.asUserException(); + } + } + } diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java index fb4d57b5f84..f14ce4141d6 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/GraphDatabaseApiProxy.java @@ -39,12 +39,8 @@ import org.neo4j.kernel.impl.coreapi.InternalTransaction; import org.neo4j.kernel.impl.factory.DbmsInfo; import org.neo4j.kernel.impl.query.Neo4jTransactionalContextFactory; -import org.neo4j.kernel.impl.query.QueryExecution; import org.neo4j.kernel.impl.query.QueryExecutionEngine; -import org.neo4j.kernel.impl.query.QueryExecutionKernelException; -import org.neo4j.kernel.impl.query.QuerySubscriber; import org.neo4j.kernel.impl.query.TransactionalContextFactory; -import org.neo4j.kernel.impl.util.ValueUtils; import org.neo4j.kernel.internal.GraphDatabaseAPI; import java.util.Map; @@ -142,23 +138,6 @@ public static Result runQueryWithoutClosingTheResult(Transaction tx, String quer return tx.execute(query, params); } - public static QueryExecution runQueryWithoutClosingTheResult( - InternalTransaction tx, - String query, - Map params, - TransactionalContextFactory contextFactory, - QueryExecutionEngine executionEngine, - QuerySubscriber subscriber - ) { - var convertedParams = ValueUtils.asMapValue(params); - var context = contextFactory.newContext(tx, query, convertedParams); - try { - return executionEngine.executeQuery(query, convertedParams, context, false, subscriber); - } catch (QueryExecutionKernelException e) { - throw e.asUserException(); - } - } - public static Result runQueryWithoutClosingTheResult( KernelTransaction tx, String query, From dd77e7eedd14ec0a1eefc7887bac6b0a4ffd86ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 26 Jan 2023 10:50:36 +0100 Subject: [PATCH 110/400] Update 54 compat --- .../CallableUserAggregationFunctionImpl.java | 77 +++++++++++++++++++ .../neo4j/gds/compat/_54/Neo4jProxyImpl.java | 8 ++ .../InMemoryStorageReader54.java | 16 ++-- 3 files changed, 90 insertions(+), 11 deletions(-) create mode 100644 compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableUserAggregationFunctionImpl.java diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableUserAggregationFunctionImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..aa1cabf9039 --- /dev/null +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index 7e97c5697e1..ef6f3408a53 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -37,6 +37,7 @@ import org.neo4j.gds.compat.CompatExecutionMonitor; import org.neo4j.gds.compat.CompatIndexQuery; import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; import org.neo4j.gds.compat.CompositeNodeCursor; import org.neo4j.gds.compat.CustomAccessMode; import org.neo4j.gds.compat.GdsDatabaseLayout; @@ -111,6 +112,7 @@ import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.KernelTransactionHandle; import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; import org.neo4j.kernel.database.NamedDatabaseId; import org.neo4j.kernel.database.NormalizedDatabaseName; import org.neo4j.kernel.database.TestDatabaseIdRepository; @@ -891,6 +893,12 @@ public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { return new CallableProcedureImpl(procedure); } + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + @Override public long transactionId(KernelTransactionHandle kernelTransactionHandle) { return kernelTransactionHandle.getTransactionSequenceNumber(); diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java index d5ddf2ed531..3b2d28616fd 100644 --- a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java +++ b/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java @@ -281,25 +281,19 @@ public int propertyKeyCount() { int nodePropertyCount = graphStore .schema() .nodeSchema() - .properties() - .values() - .stream() - .mapToInt(map -> map.keySet().size()) - .sum(); + .allProperties() + .size(); int relPropertyCount = graphStore .schema() .relationshipSchema() - .properties() - .values() - .stream() - .mapToInt(map -> map.keySet().size()) - .sum(); + .allProperties() + .size(); return nodePropertyCount + relPropertyCount; } @Override public int relationshipTypeCount() { - return graphStore.schema().relationshipSchema().properties().keySet().size(); + return graphStore.schema().relationshipSchema().availableTypes().size(); } @Override From 1575caebdae9aeaa0184950084468df92765883d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 26 Jan 2023 11:17:49 +0100 Subject: [PATCH 111/400] Fix inputRelationship count returned in ToUndirected --- doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc | 2 +- .../java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc b/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc index ee502d00e13..1e1d53454b8 100644 --- a/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc +++ b/doc/modules/ROOT/pages/graph-catalog-relationship-ops.adoc @@ -685,7 +685,7 @@ YIELD [opts="header"] |=== | inputRelationships | relationshipsWritten -| 19 | 18 +| 9 | 18 |=== -- diff --git a/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java b/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java index 5d721bf6cc6..da70a66c493 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java +++ b/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java @@ -65,7 +65,9 @@ protected AbstractResultBuilder resultBuilder( ComputationResult computeResult, ExecutionContext executionContext ) { - return new MutateResult.Builder().withInputRelationships(computeResult.graph().relationshipCount()); + return new MutateResult.Builder().withInputRelationships(computeResult + .graphStore() + .relationshipCount(RelationshipType.of(computeResult.config().relationshipType()))); } @Override From 8e6b6bef979d82d347b0d761dec3baf8eab2c9ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 26 Jan 2023 15:10:37 +0100 Subject: [PATCH 112/400] Document Neo4j 5.4 support --- README.adoc | 8 +++++--- .../ROOT/pages/installation/supported-neo4j-versions.adoc | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.adoc b/README.adoc index 5131f55733e..62dffaed601 100644 --- a/README.adoc +++ b/README.adoc @@ -75,18 +75,20 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 4.3.0 - 4.3.18 |Neo4j 4.4.0 - 4.4.11 -.4+<.^|GDS 2.2.x +.6+<.^|GDS 2.2.x |Neo4j 4.3.15 - 4.3.23 -.7+.^|Java 11 & Java 17 +.11+.^|Java 11 & Java 17 |Neo4j 4.4.9 - 4.4.16 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 -.3+<.^|GDS 2.3.x +|Neo4j 5.4.0 +.5+<.^|GDS 2.3.x |Neo4j 4.4.9 - 4.4.16 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 +|Neo4j 5.4.0 |=== NOTE: Preview releases are not automatically made available in Neo4j Desktop. They need to be installed manually. diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 3f78436afce..8dba7e593a9 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,6 +10,7 @@ Time to upgrade! [opts=header] |=== | Neo4j version | Neo4j Graph Data Science +| `5.4` | `2.3`, `2.2.7 or later` | `5.3` | `2.3`, `2.2.6 or later` | `5.2` | `2.3`, `2.2.3 or later` | `5.1`| `2.3`, `2.2.1` From 52c132e3ed3d201dde737d21030b97d37da08be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 26 Jan 2023 16:59:41 +0100 Subject: [PATCH 113/400] Bump AuraVersion Co-authored-by: Yuval Rotenberg --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index b8824f463af..d556eb7a6ec 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.0' - gdsAuraVersion = '9' + gdsAuraVersion = '10' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 89a8b8dd51c1c0cdd6e5328b2027fe0ad31423a6 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Thu, 26 Jan 2023 16:24:55 +0000 Subject: [PATCH 114/400] Fix Leiden doc --- doc/modules/ROOT/pages/algorithms/leiden.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/leiden.adoc b/doc/modules/ROOT/pages/algorithms/leiden.adoc index fe642ed62ad..8180b0b1185 100644 --- a/doc/modules/ROOT/pages/algorithms/leiden.adoc +++ b/doc/modules/ROOT/pages/algorithms/leiden.adoc @@ -1,4 +1,4 @@ -le[[algorithms-leiden]] +[[algorithms-leiden]] [.beta] = Leiden :description: This section describes the Leiden algorithm in the Neo4j Graph Data Science library. From 8be5eda0f46fe269f2f8c5a9a25b88023927a482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 26 Jan 2023 17:25:16 +0100 Subject: [PATCH 115/400] Bump GDS version to 2.3.1 Co-authored-by: Yuval Rotenberg --- README.adoc | 16 ++++++++-------- .../pages/management-ops/utility-functions.adoc | 2 +- gradle/version.gradle | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.adoc b/README.adoc index 62dffaed601..12e820c7d5b 100644 --- a/README.adoc +++ b/README.adoc @@ -132,7 +132,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.2.6 + 2.3.0 ---- @@ -144,21 +144,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.2.6 + 2.3.0 org.neo4j.gds algo - 2.2.6 + 2.3.0 org.neo4j.gds alpha-algo - 2.2.6 + 2.3.0 ---- @@ -170,28 +170,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.2.6 + 2.3.0 org.neo4j.gds proc - 2.2.6 + 2.3.0 org.neo4j.gds alpha-proc - 2.2.6 + 2.3.0 org.neo4j.gds write-services - 2.2.6 + 2.3.0 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index a4f84b26c78..4bd288497cc 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.0" +| "2.3.1" |=== -- diff --git a/gradle/version.gradle b/gradle/version.gradle index d556eb7a6ec..131fb7fd0a7 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.3.0' + gdsBaseVersion = '2.3.1' gdsAuraVersion = '10' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From ecba5dce89225e36b284aacd658f9f2fc3c32458 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 26 Jan 2023 14:32:10 +0100 Subject: [PATCH 116/400] Expose bug & fix --- .../java/org/neo4j/gds/louvain/Louvain.java | 21 ++-------- .../gds/modularity/ModularityCalculator.java | 2 +- .../org/neo4j/gds/louvain/LouvainTest.java | 40 +++++++++++++++++++ 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java b/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java index 62b15fcad94..7bfbd9528c8 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java @@ -229,10 +229,7 @@ private Graph summarizeGraph( }); terminationFlag.assertRunning(); - double scaleCoefficient = 1.0; - if (workingGraph.schema().isUndirected()) { - scaleCoefficient /= 2.0; - } + IdMap idMap = nodesBuilder.build().idMap(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) @@ -244,7 +241,6 @@ private Graph summarizeGraph( .executorService(executorService) .build(); - double finalScaleCoefficient = scaleCoefficient; var relationshipCreators = PartitionUtils.rangePartition( concurrency, workingGraph.nodeCount(), @@ -253,7 +249,6 @@ private Graph summarizeGraph( relationshipsBuilder, modularityOptimization, workingGraph.concurrentCopy(), - finalScaleCoefficient, partition ), Optional.empty() @@ -315,21 +310,16 @@ static final class RelationshipCreator implements Runnable { private final Partition partition; - private final double scaleCoefficient; - - private RelationshipCreator( RelationshipsBuilder relationshipsBuilder, ModularityOptimization modularityOptimization, RelationshipIterator relationshipIterator, - double scaleCoefficient, Partition partition ) { this.relationshipsBuilder = relationshipsBuilder; this.modularityOptimization = modularityOptimization; this.relationshipIterator = relationshipIterator; this.partition = partition; - this.scaleCoefficient = scaleCoefficient; } @Override @@ -337,18 +327,13 @@ public void run() { partition.consume(nodeId -> { long communityId = modularityOptimization.getCommunityId(nodeId); relationshipIterator.forEachRelationship(nodeId, 1.0, (source, target, property) -> { - // In the case of undirected graphs, we need scaling as otherwise we'd have double the value in these edges - // see GraphAggregationPhase.java for the equivalent in Leiden; and the corresponding test - if (source == target) { - relationshipsBuilder.add(communityId, modularityOptimization.getCommunityId(target), property); - } else { + //ignore scaling alltogether relationshipsBuilder.add( communityId, modularityOptimization.getCommunityId(target), - property * scaleCoefficient + property ); - } return true; }); diff --git a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java index b9e6ed11e2f..5048b1ca9b8 100644 --- a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java +++ b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java @@ -41,7 +41,7 @@ public class ModularityCalculator extends Algorithm { private final int concurrency; - protected ModularityCalculator( + public ModularityCalculator( Graph graph, LongUnaryOperator communityIdProvider, int concurrency diff --git a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java index 9bb46d93aa6..cb2df34ecf6 100644 --- a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java +++ b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.louvain; +import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -27,9 +28,12 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.beta.generator.RandomGraphGenerator; import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.config.RandomGraphGeneratorConfig; +import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; import org.neo4j.gds.core.concurrency.Pools; @@ -45,11 +49,14 @@ import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.modularity.ModularityCalculator; import java.util.Map; import java.util.Optional; +import java.util.function.LongUnaryOperator; import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -516,6 +523,39 @@ void shouldThrowOnNegativeSeed() { assertThatThrownBy(algorithm::compute).hasMessageContaining("non-negative"); + } + + @Test + void shouldGiveSameResultWithCalculator() { + var myGraph = RandomGraphGenerator + .builder() + .nodeCount(1_000) + .averageDegree(10) + .relationshipDistribution(RelationshipDistribution.UNIFORM) + .direction(Direction.UNDIRECTED) + .allowSelfLoops(RandomGraphGeneratorConfig.AllowSelfLoops.YES) + .aggregation(Aggregation.SINGLE) + .seed(42) + .build() + .generate(); + + var louvain = new Louvain( + myGraph, + ImmutableLouvainStreamConfig.builder().build(), + false, + 10, + 10, + TOLERANCE_DEFAULT, + 4, + ProgressTracker.NULL_TRACKER, + Pools.DEFAULT + ); + var result = louvain.compute(); + assertThat(louvain.levels()).isGreaterThan(1); + LongUnaryOperator vToCommunity = v -> result.getCommunity(v); + var modularityCalculator = new ModularityCalculator(myGraph, vToCommunity, 4); + double calculatedModularity = modularityCalculator.compute().totalModularity(); + assertThat(result.modularities()[louvain.levels() - 1]).isCloseTo(calculatedModularity, Offset.offset(1e-5)); } } From e2fed2f4e24e0b032af30f70d5d11f62f11991a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Fri, 27 Jan 2023 15:18:24 +0100 Subject: [PATCH 117/400] Set Scala version for 5.5 --- gradle/dependencies.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index e4bbe89bde5..2687d587310 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -18,6 +18,7 @@ ext { '5.2': '2.13.8', '5.3': '2.13.8', '5.4': '2.13.8', + '5.5': '2.13.8', ] ver = [ From e663572616f35b1695c4b897cb6e9d8760fe96ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Fri, 27 Jan 2023 15:47:19 +0100 Subject: [PATCH 118/400] Override semanticVersion for RC --- .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index c8550611b9d..6dcf3a1d681 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -55,6 +55,9 @@ public String toString() { } public MajorMinorVersion semanticVersion() { + if (this == V_RC) { + return ImmutableMajorMinorVersion.of(5, 5); + } String version = toString(); var subVersions = version.split("\\."); From 97442e70aac4bf6df62a49ceccc3856d81d011bb Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Mon, 30 Jan 2023 11:28:55 +0100 Subject: [PATCH 119/400] List HashGNN as beta in docs --- .../ROOT/pages/machine-learning/node-embeddings/index.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/index.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/index.adoc index 9cb9a082d4d..b672a3ebb0c 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/index.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/index.adoc @@ -13,8 +13,6 @@ The Neo4j Graph Data Science library contains the following node embedding algor * Beta ** xref:machine-learning/node-embeddings/graph-sage.adoc[GraphSAGE] ** xref:machine-learning/node-embeddings/node2vec.adoc[Node2Vec] - -* Alpha ** xref:machine-learning/node-embeddings/hashgnn.adoc[HashGNN] From dbedd1ce7170cdef06cca39ec91724762992e25f Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Mon, 30 Jan 2023 11:45:41 +0000 Subject: [PATCH 120/400] Update the link to strict validation setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre Co-authored-by: Lidia Zuin --- doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 657541c98a3..39fde81e854 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -42,7 +42,7 @@ For more information on setting up, configuring and managing a Neo4j cluster, pl [NOTE] ====== -When working with cluster configuration you should beware https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_dbms.config.strict_validation[strict config validation] in Neo4j. +When working with cluster configuration you should beware https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_server.config.strict_validation.enabled[strict config validation] in Neo4j. When configuring GDS for a Read Replica you will introduce GDS-specific configuration into `neo4j.conf` - and that is fine because with the GDS plugin installed, Neo4j will happily validate those configuration items. From 0a83c892956457bfec40720963302fd928b4ba86 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 26 Jan 2023 16:14:21 +0100 Subject: [PATCH 121/400] Allow passing token to label mapping to NodeImporter --- .../neo4j/gds/core/loading/NodeImporter.java | 42 ++++++++++++------- .../core/loading/ScanningNodesImporter.java | 12 +++--- .../loading/construction/NodesBuilder.java | 12 +++--- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NodeImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/NodeImporter.java index de38ea8fbe1..729f9ace438 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NodeImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NodeImporter.java @@ -20,12 +20,14 @@ package org.neo4j.gds.core.loading; import com.carrotsearch.hppc.IntObjectMap; +import org.immutables.builder.Builder; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.compat.PropertyReference; import org.neo4j.gds.core.utils.RawValues; import java.util.Collections; import java.util.List; +import java.util.Optional; public class NodeImporter { @@ -35,13 +37,14 @@ public interface PropertyReader { private final IdMapBuilder idMapBuilder; private final LabelInformation.Builder labelInformationBuilder; - private final IntObjectMap> labelTokenNodeLabelMapping; + private final Optional>> labelTokenNodeLabelMapping; private final boolean importProperties; - public NodeImporter( + @Builder.Constructor + NodeImporter( IdMapBuilder idMapBuilder, LabelInformation.Builder labelInformationBuilder, - IntObjectMap> labelTokenNodeLabelMapping, + Optional>> labelTokenNodeLabelMapping, boolean importProperties ) { this.idMapBuilder = idMapBuilder; @@ -51,6 +54,13 @@ public NodeImporter( } public long importNodes(NodesBatchBuffer buffer, PropertyReader reader) { + var tokenToLabelMap = this.labelTokenNodeLabelMapping.orElseThrow( + () -> new IllegalStateException("Missing Token-to-NodeLabel mapping") + ); + return importNodes(buffer, tokenToLabelMap, reader); + } + + public long importNodes(NodesBatchBuffer buffer, IntObjectMap> tokenToNodeLabelsMap, PropertyReader reader) { int batchLength = buffer.length(); if (batchLength == 0) { return 0; @@ -84,7 +94,8 @@ public long importNodes(NodesBatchBuffer buffer, PropertyReader reader) { batch, batchLength, labelIds, - (nodeIds, pos) -> nodeIds[pos] + (nodeIds, pos) -> nodeIds[pos], + tokenToNodeLabelsMap ); } @@ -96,19 +107,22 @@ public long importNodes(NodesBatchBuffer buffer, PropertyReader reader) { return RawValues.combineIntInt(batchLength, importedProperties); } - private void setNodeLabelInformation(long[] batch, int batchLength, long[][] labelIds, IdFunction idFunction) { + private void setNodeLabelInformation( + long[] batch, + int batchLength, + long[][] labelIds, + IdFunction idFunction, + IntObjectMap> tokenToNodeLabelsMap + ) { int cappedBatchLength = Math.min(labelIds.length, batchLength); for (int i = 0; i < cappedBatchLength; i++) { long nodeId = idFunction.apply(batch, i); - long[] labelIdsForNode = labelIds[i]; - - for (long labelId : labelIdsForNode) { - var elementIdentifiers = labelTokenNodeLabelMapping.getOrDefault( - (int) labelId, - Collections.emptyList() - ); - for (NodeLabel elementIdentifier : elementIdentifiers) { - labelInformationBuilder.addNodeIdToLabel(elementIdentifier, nodeId); + long[] labelTokensForNode = labelIds[i]; + + for (long token : labelTokensForNode) { + var nodeLabels = tokenToNodeLabelsMap.getOrDefault((int) token, Collections.emptyList()); + for (NodeLabel nodeLabel : nodeLabels) { + this.labelInformationBuilder.addNodeIdToLabel(nodeLabel, nodeId); } } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 36f6a09d71a..a1baec8dcb9 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -164,12 +164,12 @@ public RecordScannerTaskRunner.RecordScannerTaskFactory recordScannerTaskFactory ImportSizing sizing, StoreScanner storeScanner ) { - var nodeImporter = new NodeImporter( - idMapBuilder, - labelInformationBuilder, - dimensions.tokenNodeLabelMapping(), - nodePropertyImporter != null - ); + var nodeImporter = new NodeImporterBuilder() + .idMapBuilder(idMapBuilder) + .labelInformationBuilder(labelInformationBuilder) + .labelTokenNodeLabelMapping(dimensions.tokenNodeLabelMapping()) + .importProperties(nodePropertyImporter != null) + .build(); return NodesScannerTask.factory( transaction, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 003710fbefb..200d8efc013 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -38,6 +38,7 @@ import org.neo4j.gds.core.loading.LabelInformation; import org.neo4j.gds.core.loading.LabelInformationBuilders; import org.neo4j.gds.core.loading.NodeImporter; +import org.neo4j.gds.core.loading.NodeImporterBuilder; import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.NodesBatchBuffer; import org.neo4j.gds.core.loading.NodesBatchBufferBuilder; @@ -104,12 +105,11 @@ public final class NodesBuilder { : LabelInformationBuilders.multiLabelWithCapacity(maxOriginalId + 1); this.propertyBuildersByPropertyKey = propertyBuildersByPropertyKey; this.importedNodes = new LongAdder(); - this.nodeImporter = new NodeImporter( - idMapBuilder, - labelInformationBuilder, - tokenToNodeLabel.labelTokenNodeLabelMapping(), - hasProperties - ); + this.nodeImporter = new NodeImporterBuilder() + .idMapBuilder(idMapBuilder) + .labelInformationBuilder(labelInformationBuilder) + .importProperties(hasProperties) + .build(); Function propertyBuilderFn = propertyBuildersByPropertyKey.isEmpty() ? this::getOrCreatePropertyBuilder From 2e4a62c358d181728b8b2458b388608d9df86961 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 26 Jan 2023 16:14:47 +0100 Subject: [PATCH 122/400] Use thread-local TokenToNodeLabels in NodesBuilder --- .../loading/construction/GraphFactory.java | 23 +--- .../loading/construction/NodesBuilder.java | 21 ++-- .../construction/TokenToNodeLabel.java | 110 ---------------- .../construction/TokenToNodeLabels.java | 117 ++++++++++++++++++ 4 files changed, 130 insertions(+), 141 deletions(-) delete mode 100644 core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabel.java create mode 100644 core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabels.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 1e297e31e14..b881192d386 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -19,13 +19,9 @@ */ package org.neo4j.gds.core.loading.construction; -import com.carrotsearch.hppc.IntObjectHashMap; -import com.carrotsearch.hppc.ObjectIntScatterMap; -import org.apache.commons.lang3.mutable.MutableInt; import org.immutables.builder.Builder; import org.immutables.value.Value; import org.neo4j.gds.ImmutableRelationshipProjection; -import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; @@ -61,7 +57,6 @@ import java.util.stream.IntStream; import static java.util.stream.Collectors.toMap; -import static org.neo4j.gds.core.GraphDimensions.ANY_LABEL; import static org.neo4j.kernel.api.StatementConstants.NO_SUCH_RELATIONSHIP_TYPE; @Value.Style( @@ -120,7 +115,7 @@ static NodesBuilder nodesBuilder( )).orElseGet(() -> new NodesBuilder( maxOriginalId, threadCount, - TokenToNodeLabel.lazy(), + TokenToNodeLabels::lazy, NodeLabelTokenToPropertyKeys::lazy, new ConcurrentHashMap<>(), idMapBuilder, @@ -139,20 +134,6 @@ private static NodesBuilder fromSchema( boolean hasLabelInformation, boolean deduplicateIds ) { - var nodeLabels = nodeSchema.availableLabels(); - - var elementIdentifierLabelTokenMapping = new ObjectIntScatterMap(); - var labelTokenNodeLabelMapping = new IntObjectHashMap>(); - var labelTokenCounter = new MutableInt(0); - nodeLabels.forEach(nodeLabel -> { - int labelToken = nodeLabel == NodeLabel.ALL_NODES - ? ANY_LABEL - : labelTokenCounter.getAndIncrement(); - - elementIdentifierLabelTokenMapping.put(nodeLabel, labelToken); - labelTokenNodeLabelMapping.put(labelToken, List.of(nodeLabel)); - }); - var propertyBuildersByPropertyKey = nodeSchema.unionProperties().entrySet().stream().collect(toMap( Map.Entry::getKey, e -> NodePropertiesFromStoreBuilder.of(e.getValue().defaultValue(), concurrency) @@ -161,7 +142,7 @@ private static NodesBuilder fromSchema( return new NodesBuilder( maxOriginalId, concurrency, - TokenToNodeLabel.fixed(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping), + () -> TokenToNodeLabels.fixed(nodeSchema.availableLabels()), () -> NodeLabelTokenToPropertyKeys.fixed(nodeSchema), new ConcurrentHashMap<>(propertyBuildersByPropertyKey), idMapBuilder, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index 200d8efc013..a604c798522 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -67,7 +67,7 @@ public final class NodesBuilder { - public static final DefaultValue NO_PROPERTY_VALUE = DefaultValue.DEFAULT; + private static final DefaultValue NO_PROPERTY_VALUE = DefaultValue.DEFAULT; public static final long UNKNOWN_MAX_ID = -1L; private final long maxOriginalId; @@ -87,7 +87,7 @@ public final class NodesBuilder { NodesBuilder( long maxOriginalId, int concurrency, - TokenToNodeLabel tokenToNodeLabel, + Supplier tokenToNodeLabelSupplier, Supplier nodeLabelTokenToPropertyKeysSupplier, ConcurrentMap propertyBuildersByPropertyKey, IdMapBuilder idMapBuilder, @@ -126,7 +126,7 @@ public final class NodesBuilder { seenNodeIdPredicate, hasLabelInformation, hasProperties, - tokenToNodeLabel, + tokenToNodeLabelSupplier.get(), nodeLabelTokenToPropertyKeysSupplier.get(), propertyBuilderFn ) @@ -286,7 +286,7 @@ private static NodeProperty entryToNodeProperty( * Closes the NodesBuilder without flushing the internal buffers. * The given exception is thrown, once the thread local builders * are closed. - * + *

* This method must be called in case of an error while using the * NodesBuilder. */ @@ -320,7 +320,7 @@ private static class ThreadLocalBuilder implements AutoCloseable { private final LongAdder importedNodes; private final LongPredicate seenNodeIdPredicate; - private final TokenToNodeLabel tokenToNodeLabel; + private final TokenToNodeLabels tokenToNodeLabels; private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; private final NodesBatchBuffer buffer; private final Function propertyBuilderFn; @@ -334,13 +334,13 @@ private static class ThreadLocalBuilder implements AutoCloseable { LongPredicate seenNodeIdPredicate, boolean hasLabelInformation, boolean hasProperties, - TokenToNodeLabel tokenToNodeLabel, + TokenToNodeLabels tokenToNodeLabels, NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, Function propertyBuilderFn ) { this.importedNodes = importedNodes; this.seenNodeIdPredicate = seenNodeIdPredicate; - this.tokenToNodeLabel = tokenToNodeLabel; + this.tokenToNodeLabels = tokenToNodeLabels; this.nodeLabelTokenToPropertyKeys = nodeLabelTokenToPropertyKeys; this.propertyBuilderFn = propertyBuilderFn; @@ -389,7 +389,7 @@ private long[] getOrCreateLabelTokens(NodeLabelToken nodeLabels) { long[] labelIds = new long[nodeLabels.size()]; for (int i = 0; i < labelIds.length; i++) { - labelIds[i] = tokenToNodeLabel.getOrCreateToken(nodeLabels.get(i)); + labelIds[i] = this.tokenToNodeLabels.getOrCreateToken(nodeLabels.get(i)); } return labelIds; @@ -402,7 +402,8 @@ public void flush() { private void flushBuffer() { var importedNodesAndProperties = this.nodeImporter.importNodes( - buffer, + this.buffer, + this.tokenToNodeLabels.labelTokenNodeLabelMapping(), (nodeReference, labelIds, propertiesReference) -> { if (!propertiesReference.isEmpty()) { var propertyValueIndex = (int) ((LongPropertyReference) propertiesReference).id; @@ -445,7 +446,7 @@ private int importProperty(long neoNodeId, String propertyKey, Value value) { private long[] anyLabelArray() { var anyLabelArray = this.anyLabelArray; if (anyLabelArray[0] == NOT_INITIALIZED) { - anyLabelArray[0] = tokenToNodeLabel.getOrCreateToken(NodeLabel.ALL_NODES); + anyLabelArray[0] = tokenToNodeLabels.getOrCreateToken(NodeLabel.ALL_NODES); } return anyLabelArray; } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabel.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabel.java deleted file mode 100644 index 59bc6b8dc0d..00000000000 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabel.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.core.loading.construction; - -import com.carrotsearch.hppc.IntObjectHashMap; -import com.carrotsearch.hppc.ObjectIntMap; -import com.carrotsearch.hppc.ObjectIntScatterMap; -import org.neo4j.gds.NodeLabel; - -import java.util.Collections; -import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import static org.neo4j.gds.core.GraphDimensions.NO_SUCH_LABEL; -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; - -abstract class TokenToNodeLabel { - - final ObjectIntMap elementIdentifierLabelTokenMapping; - final IntObjectHashMap> labelTokenNodeLabelMapping; - - static TokenToNodeLabel fixed( - ObjectIntMap elementIdentifierLabelTokenMapping, - IntObjectHashMap> labelTokenNodeLabelMapping - ) { - return new FixedTokenToNodeLabel(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); - } - - static TokenToNodeLabel lazy() { - return new LazyTokenToNodeLabel(); - } - - TokenToNodeLabel( - ObjectIntMap elementIdentifierLabelTokenMapping, - IntObjectHashMap> labelTokenNodeLabelMapping - ) { - this.elementIdentifierLabelTokenMapping = elementIdentifierLabelTokenMapping; - this.labelTokenNodeLabelMapping = labelTokenNodeLabelMapping; - } - - IntObjectHashMap> labelTokenNodeLabelMapping() { - return this.labelTokenNodeLabelMapping; - } - - abstract int getOrCreateToken(NodeLabel nodeLabel); - - private static class FixedTokenToNodeLabel extends TokenToNodeLabel { - - FixedTokenToNodeLabel( - ObjectIntMap elementIdentifierLabelTokenMapping, - IntObjectHashMap> labelTokenNodeLabelMapping - ) { - super(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); - } - - @Override - public int getOrCreateToken(NodeLabel nodeLabel) { - if (!elementIdentifierLabelTokenMapping.containsKey(nodeLabel)) { - throw new IllegalArgumentException(formatWithLocale("No token was specified for node label %s", nodeLabel)); - } - return elementIdentifierLabelTokenMapping.get(nodeLabel); - } - } - - private static class LazyTokenToNodeLabel extends TokenToNodeLabel { - - private final Lock lock; - private int nextLabelId; - - LazyTokenToNodeLabel() { - super(new ObjectIntScatterMap<>(), new IntObjectHashMap<>()); - this.lock = new ReentrantLock(true); - this.nextLabelId = 0; - } - - @Override - public int getOrCreateToken(NodeLabel nodeLabel) { - var token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); - if (token == NO_SUCH_LABEL) { - lock.lock(); - token = elementIdentifierLabelTokenMapping.getOrDefault(nodeLabel, NO_SUCH_LABEL); - if (token == NO_SUCH_LABEL) { - token = nextLabelId++; - labelTokenNodeLabelMapping.put(token, Collections.singletonList(nodeLabel)); - elementIdentifierLabelTokenMapping.put(nodeLabel, token); - } - lock.unlock(); - } - return token; - } - } -} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabels.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabels.java new file mode 100644 index 00000000000..376bd96c379 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/TokenToNodeLabels.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import com.carrotsearch.hppc.IntObjectHashMap; +import com.carrotsearch.hppc.ObjectIntMap; +import com.carrotsearch.hppc.ObjectIntScatterMap; +import org.apache.commons.lang3.mutable.MutableInt; +import org.neo4j.gds.NodeLabel; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static org.neo4j.gds.core.GraphDimensions.ANY_LABEL; +import static org.neo4j.gds.core.GraphDimensions.NO_SUCH_LABEL; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +abstract class TokenToNodeLabels { + + final ObjectIntMap nodeLabelToLabelTokenMap; + final IntObjectHashMap> labelTokenToNodeLabelMap; + + static TokenToNodeLabels fixed(Collection nodeLabels) { + var elementIdentifierLabelTokenMapping = new ObjectIntScatterMap(); + var labelTokenNodeLabelMapping = new IntObjectHashMap>(); + var labelTokenCounter = new MutableInt(0); + nodeLabels.forEach(nodeLabel -> { + int labelToken = nodeLabel == NodeLabel.ALL_NODES + ? ANY_LABEL + : labelTokenCounter.getAndIncrement(); + + elementIdentifierLabelTokenMapping.put(nodeLabel, labelToken); + labelTokenNodeLabelMapping.put(labelToken, List.of(nodeLabel)); + }); + + return new Fixed(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); + } + + static TokenToNodeLabels lazy() { + return new Lazy(); + } + + private TokenToNodeLabels() { + this.nodeLabelToLabelTokenMap = new ObjectIntScatterMap<>(); + this.labelTokenToNodeLabelMap = new IntObjectHashMap<>(); + } + + private TokenToNodeLabels( + ObjectIntMap nodeLabelToLabelTokenMap, + IntObjectHashMap> labelTokenToNodeLabelMap + ) { + this.nodeLabelToLabelTokenMap = nodeLabelToLabelTokenMap; + this.labelTokenToNodeLabelMap = labelTokenToNodeLabelMap; + } + + IntObjectHashMap> labelTokenNodeLabelMapping() { + return this.labelTokenToNodeLabelMap; + } + + abstract int getOrCreateToken(NodeLabel nodeLabel); + + private static final class Fixed extends TokenToNodeLabels { + + private Fixed( + ObjectIntMap elementIdentifierLabelTokenMapping, + IntObjectHashMap> labelTokenNodeLabelMapping + ) { + super(elementIdentifierLabelTokenMapping, labelTokenNodeLabelMapping); + } + + @Override + public int getOrCreateToken(NodeLabel nodeLabel) { + if (!nodeLabelToLabelTokenMap.containsKey(nodeLabel)) { + throw new IllegalArgumentException(formatWithLocale("No token was specified for node label %s", nodeLabel)); + } + return nodeLabelToLabelTokenMap.get(nodeLabel); + } + } + + private static final class Lazy extends TokenToNodeLabels { + + private int nextLabelId; + + private Lazy() { + this.nextLabelId = 0; + } + + @Override + public int getOrCreateToken(NodeLabel nodeLabel) { + var token = nodeLabelToLabelTokenMap.getOrDefault(nodeLabel, NO_SUCH_LABEL); + if (token == NO_SUCH_LABEL) { + token = nextLabelId++; + labelTokenToNodeLabelMap.put(token, Collections.singletonList(nodeLabel)); + nodeLabelToLabelTokenMap.put(nodeLabel, token); + } + return token; + } + } +} From 6b562661cff1fc6a0bb115fe5e1dacc5668b880b Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 26 Jan 2023 16:59:59 +0100 Subject: [PATCH 123/400] Refactor mapping data structures into a context type --- .../loading/construction/GraphFactory.java | 16 +- .../loading/construction/NodesBuilder.java | 160 +++++----------- .../construction/NodesBuilderContext.java | 181 ++++++++++++++++++ 3 files changed, 234 insertions(+), 123 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index b881192d386..fe0e35d4f26 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -46,17 +46,14 @@ import org.neo4j.gds.core.loading.RecordsBatchBuffer; import org.neo4j.gds.core.loading.SingleTypeRelationshipImporterBuilder; import org.neo4j.gds.core.loading.SingleTypeRelationships; -import org.neo4j.gds.core.loading.nodeproperties.NodePropertiesFromStoreBuilder; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.stream.IntStream; -import static java.util.stream.Collectors.toMap; import static org.neo4j.kernel.api.StatementConstants.NO_SUCH_RELATIONSHIP_TYPE; @Value.Style( @@ -115,9 +112,7 @@ static NodesBuilder nodesBuilder( )).orElseGet(() -> new NodesBuilder( maxOriginalId, threadCount, - TokenToNodeLabels::lazy, - NodeLabelTokenToPropertyKeys::lazy, - new ConcurrentHashMap<>(), + NodesBuilderContext.lazy(threadCount), idMapBuilder, labelInformation, hasProperties.orElse(false), @@ -134,17 +129,10 @@ private static NodesBuilder fromSchema( boolean hasLabelInformation, boolean deduplicateIds ) { - var propertyBuildersByPropertyKey = nodeSchema.unionProperties().entrySet().stream().collect(toMap( - Map.Entry::getKey, - e -> NodePropertiesFromStoreBuilder.of(e.getValue().defaultValue(), concurrency) - )); - return new NodesBuilder( maxOriginalId, concurrency, - () -> TokenToNodeLabels.fixed(nodeSchema.availableLabels()), - () -> NodeLabelTokenToPropertyKeys.fixed(nodeSchema), - new ConcurrentHashMap<>(propertyBuildersByPropertyKey), + NodesBuilderContext.fixed(nodeSchema, concurrency), idMapBuilder, hasLabelInformation, nodeSchema.hasProperties(), diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index a604c798522..a95818e4d11 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -21,17 +21,15 @@ import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.compat.LongPropertyReference; +import org.neo4j.gds.compat.PropertyReference; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.loading.IdMapBuilder; import org.neo4j.gds.core.loading.ImmutableNodes; @@ -55,19 +53,14 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.LongAdder; import java.util.function.Function; import java.util.function.LongPredicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.stream.Collectors.toMap; public final class NodesBuilder { - - private static final DefaultValue NO_PROPERTY_VALUE = DefaultValue.DEFAULT; public static final long UNKNOWN_MAX_ID = -1L; private final long maxOriginalId; @@ -78,18 +71,16 @@ public final class NodesBuilder { private final LabelInformation.Builder labelInformationBuilder; private final LongAdder importedNodes; - private final AutoCloseableThreadLocal threadLocalBuilder; + private final AutoCloseableThreadLocal threadLocalBuilders; private final NodeImporter nodeImporter; - private final ConcurrentMap propertyBuildersByPropertyKey; + private final NodesBuilderContext nodesBuilderContext; NodesBuilder( long maxOriginalId, int concurrency, - Supplier tokenToNodeLabelSupplier, - Supplier nodeLabelTokenToPropertyKeysSupplier, - ConcurrentMap propertyBuildersByPropertyKey, + NodesBuilderContext nodesBuilderContext, IdMapBuilder idMapBuilder, boolean hasLabelInformation, boolean hasProperties, @@ -98,12 +89,13 @@ public final class NodesBuilder { ) { this.maxOriginalId = maxOriginalId; this.concurrency = concurrency; + this.nodesBuilderContext = nodesBuilderContext; this.idMapBuilder = idMapBuilder; this.propertyStates = propertyStates; this.labelInformationBuilder = !hasLabelInformation ? LabelInformationBuilders.allNodes() : LabelInformationBuilders.multiLabelWithCapacity(maxOriginalId + 1); - this.propertyBuildersByPropertyKey = propertyBuildersByPropertyKey; + this.importedNodes = new LongAdder(); this.nodeImporter = new NodeImporterBuilder() .idMapBuilder(idMapBuilder) @@ -111,14 +103,12 @@ public final class NodesBuilder { .importProperties(hasProperties) .build(); - Function propertyBuilderFn = propertyBuildersByPropertyKey.isEmpty() - ? this::getOrCreatePropertyBuilder - : this::getPropertyBuilder; LongPredicate seenNodeIdPredicate = seenNodesPredicate(deduplicateIds, maxOriginalId); long highestPossibleNodeCount = maxOriginalId == UNKNOWN_MAX_ID ? Long.MAX_VALUE : maxOriginalId + 1; - this.threadLocalBuilder = AutoCloseableThreadLocal.withInitial( + + this.threadLocalBuilders = AutoCloseableThreadLocal.withInitial( () -> new NodesBuilder.ThreadLocalBuilder( importedNodes, nodeImporter, @@ -126,9 +116,7 @@ public final class NodesBuilder { seenNodeIdPredicate, hasLabelInformation, hasProperties, - tokenToNodeLabelSupplier.get(), - nodeLabelTokenToPropertyKeysSupplier.get(), - propertyBuilderFn + nodesBuilderContext.threadLocalContext() ) ); } @@ -155,7 +143,7 @@ public void addNode(long originalId) { } public void addNode(long originalId, NodeLabelToken nodeLabels) { - this.threadLocalBuilder.get().addNode(originalId, nodeLabels); + this.threadLocalBuilders.get().addNode(originalId, nodeLabels); } public void addNode(long originalId, NodeLabel... nodeLabels) { @@ -187,7 +175,7 @@ public void addNode(long originalId, Map properties, NodeLabel no } public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues properties) { - this.threadLocalBuilder.get().addNode(originalId, nodeLabels, properties); + this.threadLocalBuilders.get().addNode(originalId, nodeLabels, properties); } public long importedNodes() { @@ -215,12 +203,14 @@ public Nodes build(long highestNeoId) { private List closeThreadLocalBuilders() { // Flush remaining buffer contents - this.threadLocalBuilder.forEach(ThreadLocalBuilder::flush); + this.threadLocalBuilders.forEach(ThreadLocalBuilder::flush); // Collect token to property keys for final union var labelTokenToPropertyKeys = new ArrayList(); - this.threadLocalBuilder.forEach(tlb -> labelTokenToPropertyKeys.add(tlb.nodeLabelTokenToPropertyKeys)); + this.threadLocalBuilders.forEach( + threadLocalBuilder -> labelTokenToPropertyKeys.add(threadLocalBuilder.threadLocalContext.nodeLabelTokenToPropertyKeys()) + ); // Clean up resources held by local builders - this.threadLocalBuilder.close(); + this.threadLocalBuilders.close(); return labelTokenToPropertyKeys; } @@ -230,7 +220,6 @@ private NodeSchema buildNodeSchema( Collection localLabelTokenToPropertyKeys, Map nodeProperties ) { - // Collect the property schemas from the imported property values. var propertyKeysToSchema = nodeProperties .entrySet() @@ -263,7 +252,7 @@ private NodeSchema buildNodeSchema( } private Map buildProperties(IdMap idMap) { - return propertyBuildersByPropertyKey.entrySet().stream().collect(toMap( + return this.nodesBuilderContext.nodePropertyBuilders().entrySet().stream().collect(toMap( Map.Entry::getKey, entry -> entryToNodeProperty(entry, propertyStates.apply(entry.getKey()), idMap) )); @@ -291,41 +280,18 @@ private static NodeProperty entryToNodeProperty( * NodesBuilder. */ public void close(RuntimeException exception) { - this.threadLocalBuilder.close(); + this.threadLocalBuilders.close(); throw exception; } - private NodePropertiesFromStoreBuilder getOrCreatePropertyBuilder(String propertyKey) { - return propertyBuildersByPropertyKey.computeIfAbsent( - propertyKey, - __ -> NodePropertiesFromStoreBuilder.of(NO_PROPERTY_VALUE, concurrency) - ); - } - - private NodePropertiesFromStoreBuilder getPropertyBuilder(String propertyKey) { - return propertyBuildersByPropertyKey.get(propertyKey); - } - - @ValueClass - public interface IdMapAndProperties { - IdMap idMap(); - - Optional> nodeProperties(); - } - private static class ThreadLocalBuilder implements AutoCloseable { - private static final long NOT_INITIALIZED = -42L; - private final long[] anyLabelArray = {NOT_INITIALIZED}; - private final LongAdder importedNodes; private final LongPredicate seenNodeIdPredicate; - private final TokenToNodeLabels tokenToNodeLabels; - private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; private final NodesBatchBuffer buffer; - private final Function propertyBuilderFn; private final NodeImporter nodeImporter; private final List batchNodeProperties; + private final NodesBuilderContext.ThreadLocalContext threadLocalContext; ThreadLocalBuilder( LongAdder importedNodes, @@ -334,15 +300,11 @@ private static class ThreadLocalBuilder implements AutoCloseable { LongPredicate seenNodeIdPredicate, boolean hasLabelInformation, boolean hasProperties, - TokenToNodeLabels tokenToNodeLabels, - NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, - Function propertyBuilderFn + NodesBuilderContext.ThreadLocalContext threadLocalContext ) { this.importedNodes = importedNodes; this.seenNodeIdPredicate = seenNodeIdPredicate; - this.tokenToNodeLabels = tokenToNodeLabels; - this.nodeLabelTokenToPropertyKeys = nodeLabelTokenToPropertyKeys; - this.propertyBuilderFn = propertyBuilderFn; + this.threadLocalContext = threadLocalContext; this.buffer = new NodesBatchBufferBuilder() .capacity(ParallelUtil.DEFAULT_BATCH_SIZE) @@ -350,15 +312,16 @@ private static class ThreadLocalBuilder implements AutoCloseable { .hasLabelInformation(hasLabelInformation) .readProperty(hasProperties) .build(); + this.nodeImporter = nodeImporter; this.batchNodeProperties = new ArrayList<>(buffer.capacity()); } - public void addNode(long originalId, NodeLabelToken nodeLabels) { + public void addNode(long originalId, NodeLabelToken nodeLabelToken) { if (!seenNodeIdPredicate.test(originalId)) { - long[] labels = getOrCreateLabelTokens(nodeLabels); + long[] threadLocalTokens = threadLocalContext.addNodeLabelToken(nodeLabelToken); - buffer.add(originalId, LongPropertyReference.empty(), labels); + buffer.add(originalId, LongPropertyReference.empty(), threadLocalTokens); if (buffer.isFull()) { flushBuffer(); reset(); @@ -366,15 +329,16 @@ public void addNode(long originalId, NodeLabelToken nodeLabels) { } } - public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues properties) { + public void addNode(long originalId, NodeLabelToken nodeLabelToken, PropertyValues properties) { if (!seenNodeIdPredicate.test(originalId)) { - long[] labels = getOrCreateLabelTokens(nodeLabels); - this.nodeLabelTokenToPropertyKeys.add(nodeLabels, properties.propertyKeys()); - + long[] threadLocalTokens = threadLocalContext.addNodeLabelTokenAndPropertyKeys( + nodeLabelToken, + properties.propertyKeys() + ); int propertyReference = batchNodeProperties.size(); batchNodeProperties.add(properties); - buffer.add(originalId, LongPropertyReference.of(propertyReference), labels); + buffer.add(originalId, LongPropertyReference.of(propertyReference), threadLocalTokens); if (buffer.isFull()) { flushBuffer(); reset(); @@ -382,19 +346,6 @@ public void addNode(long originalId, NodeLabelToken nodeLabels, PropertyValues p } } - private long[] getOrCreateLabelTokens(NodeLabelToken nodeLabels) { - if (nodeLabels.isEmpty()) { - return anyLabelArray(); - } - - long[] labelIds = new long[nodeLabels.size()]; - for (int i = 0; i < labelIds.length; i++) { - labelIds[i] = this.tokenToNodeLabels.getOrCreateToken(nodeLabels.get(i)); - } - - return labelIds; - } - public void flush() { flushBuffer(); reset(); @@ -403,21 +354,8 @@ public void flush() { private void flushBuffer() { var importedNodesAndProperties = this.nodeImporter.importNodes( this.buffer, - this.tokenToNodeLabels.labelTokenNodeLabelMapping(), - (nodeReference, labelIds, propertiesReference) -> { - if (!propertiesReference.isEmpty()) { - var propertyValueIndex = (int) ((LongPropertyReference) propertiesReference).id; - var properties = batchNodeProperties.get(propertyValueIndex); - var importedProperties = new MutableInt(0); - properties.forEach((propertyKey, propertyValue) -> importedProperties.add(importProperty( - nodeReference, - propertyKey, - propertyValue - ))); - return importedProperties.intValue(); - } - return 0; - } + this.threadLocalContext.threadLocalTokenToNodeLabels(), + this::importProperties ); int importedNodes = RawValues.getHead(importedNodesAndProperties); this.importedNodes.add(importedNodes); @@ -431,24 +369,28 @@ private void reset() { @Override public void close() {} - private int importProperty(long neoNodeId, String propertyKey, Value value) { - int propertiesImported = 0; - - var nodePropertyBuilder = propertyBuilderFn.apply(propertyKey); - if (nodePropertyBuilder != null) { - nodePropertyBuilder.set(neoNodeId, value); - propertiesImported++; + private int importProperties(long nodeReference, long[] labelIds, PropertyReference propertiesReference) { + if (!propertiesReference.isEmpty()) { + var propertyValueIndex = (int) ((LongPropertyReference) propertiesReference).id; + var properties = batchNodeProperties.get(propertyValueIndex); + var importedProperties = new MutableInt(0); + properties.forEach((propertyKey, propertyValue) -> importedProperties.add(importProperty( + nodeReference, + propertyKey, + propertyValue + ))); + return importedProperties.intValue(); } - - return propertiesImported; + return 0; } - private long[] anyLabelArray() { - var anyLabelArray = this.anyLabelArray; - if (anyLabelArray[0] == NOT_INITIALIZED) { - anyLabelArray[0] = tokenToNodeLabels.getOrCreateToken(NodeLabel.ALL_NODES); + private int importProperty(long neoNodeId, String propertyKey, Value value) { + var nodePropertyBuilder = this.threadLocalContext.nodePropertyBuilder(propertyKey); + if (nodePropertyBuilder != null) { + nodePropertyBuilder.set(neoNodeId, value); + return 1; } - return anyLabelArray; + return 0; } } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java new file mode 100644 index 00000000000..0a85e24341f --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading.construction; + +import com.carrotsearch.hppc.IntObjectMap; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.core.loading.nodeproperties.NodePropertiesFromStoreBuilder; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; +import java.util.function.Supplier; + +import static java.util.stream.Collectors.toMap; + +final class NodesBuilderContext { + + private static final DefaultValue NO_PROPERTY_VALUE = DefaultValue.DEFAULT; + + // Thread-local mappings that can be computed independently. + private final Supplier tokenToNodeLabelSupplier; + private final Supplier nodeLabelTokenToPropertyKeysSupplier; + // Thread-global mapping as all threads need to write to the same property builders. + private final ConcurrentMap propertyKeyToPropertyBuilder; + + private final int concurrency; + + /** + * Used if no node schema information is available and needs to be inferred from the input data. + */ + static NodesBuilderContext lazy(int concurrency) { + return new NodesBuilderContext( + TokenToNodeLabels::lazy, + NodeLabelTokenToPropertyKeys::lazy, + new ConcurrentHashMap<>(), + concurrency + ); + } + + /** + * Used if a node schema is available upfront. + */ + static NodesBuilderContext fixed(NodeSchema nodeSchema, int concurrency) { + var propertyBuildersByPropertyKey = nodeSchema.unionProperties().entrySet().stream().collect(toMap( + Map.Entry::getKey, + e -> NodePropertiesFromStoreBuilder.of(e.getValue().defaultValue(), concurrency) + )); + + return new NodesBuilderContext( + () -> TokenToNodeLabels.fixed(nodeSchema.availableLabels()), + () -> NodeLabelTokenToPropertyKeys.fixed(nodeSchema), + new ConcurrentHashMap<>(propertyBuildersByPropertyKey), + concurrency + ); + } + + Map nodePropertyBuilders() { + return this.propertyKeyToPropertyBuilder; + } + + private NodesBuilderContext( + Supplier tokenToNodeLabelSupplier, + Supplier nodeLabelTokenToPropertyKeysSupplier, + ConcurrentMap propertyKeyToPropertyBuilder, + int concurrency + ) { + this.tokenToNodeLabelSupplier = tokenToNodeLabelSupplier; + this.nodeLabelTokenToPropertyKeysSupplier = nodeLabelTokenToPropertyKeysSupplier; + this.propertyKeyToPropertyBuilder = propertyKeyToPropertyBuilder; + this.concurrency = concurrency; + } + + ThreadLocalContext threadLocalContext() { + Function propertyBuilderFn = this.propertyKeyToPropertyBuilder.isEmpty() + ? this::getOrCreatePropertyBuilder + : this::getPropertyBuilder; + + return new ThreadLocalContext( + tokenToNodeLabelSupplier.get(), + nodeLabelTokenToPropertyKeysSupplier.get(), + propertyBuilderFn + ); + } + + private NodePropertiesFromStoreBuilder getOrCreatePropertyBuilder(String propertyKey) { + return this.propertyKeyToPropertyBuilder.computeIfAbsent( + propertyKey, + __ -> NodePropertiesFromStoreBuilder.of(NO_PROPERTY_VALUE, concurrency) + ); + } + + private NodePropertiesFromStoreBuilder getPropertyBuilder(String propertyKey) { + return this.propertyKeyToPropertyBuilder.get(propertyKey); + } + + static class ThreadLocalContext { + + private static final long NOT_INITIALIZED = -42L; + private final long[] anyLabelArray = {NOT_INITIALIZED}; + + private final TokenToNodeLabels tokenToNodeLabels; + + private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; + + private final Function propertyBuilderFn; + + ThreadLocalContext( + TokenToNodeLabels tokenToNodeLabels, + NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys, + Function propertyBuilderFn + ) { + this.tokenToNodeLabels = tokenToNodeLabels; + this.nodeLabelTokenToPropertyKeys = nodeLabelTokenToPropertyKeys; + this.propertyBuilderFn = propertyBuilderFn; + } + + NodePropertiesFromStoreBuilder nodePropertyBuilder(String propertyKey) { + return this.propertyBuilderFn.apply(propertyKey); + } + + NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys() { + return this.nodeLabelTokenToPropertyKeys; + } + + IntObjectMap> threadLocalTokenToNodeLabels() { + return this.tokenToNodeLabels.labelTokenNodeLabelMapping(); + } + + long[] addNodeLabelToken(NodeLabelToken nodeLabelToken) { + return getOrCreateLabelTokens(nodeLabelToken); + } + + long[] addNodeLabelTokenAndPropertyKeys(NodeLabelToken nodeLabelToken, Iterable propertyKeys) { + long[] tokens = getOrCreateLabelTokens(nodeLabelToken); + this.nodeLabelTokenToPropertyKeys.add(nodeLabelToken, propertyKeys); + return tokens; + } + + private long[] getOrCreateLabelTokens(NodeLabelToken nodeLabelToken) { + if (nodeLabelToken.isEmpty()) { + return anyLabelArray(); + } + + long[] labelIds = new long[nodeLabelToken.size()]; + for (int i = 0; i < labelIds.length; i++) { + labelIds[i] = this.tokenToNodeLabels.getOrCreateToken(nodeLabelToken.get(i)); + } + + return labelIds; + } + + private long[] anyLabelArray() { + var anyLabelArray = this.anyLabelArray; + if (anyLabelArray[0] == NOT_INITIALIZED) { + anyLabelArray[0] = tokenToNodeLabels.getOrCreateToken(NodeLabel.ALL_NODES); + } + return anyLabelArray; + } + } +} From 8dbb0090666502b0ea40d4f619d7d16a8d248632 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Thu, 26 Jan 2023 17:12:18 +0100 Subject: [PATCH 124/400] Simplify node property importing in NodesBuilder --- .../loading/construction/NodesBuilder.java | 41 ++++++++----------- .../construction/NodesBuilderContext.java | 4 +- .../loading/construction/PropertyValues.java | 12 ++++++ 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index a95818e4d11..a36eb4e64c6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.core.loading.construction; -import org.apache.commons.lang3.mutable.MutableInt; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyState; @@ -351,6 +350,11 @@ public void flush() { reset(); } + private void reset() { + buffer.reset(); + batchNodeProperties.clear(); + } + private void flushBuffer() { var importedNodesAndProperties = this.nodeImporter.importNodes( this.buffer, @@ -361,36 +365,23 @@ private void flushBuffer() { this.importedNodes.add(importedNodes); } - private void reset() { - buffer.reset(); - batchNodeProperties.clear(); - } - - @Override - public void close() {} - private int importProperties(long nodeReference, long[] labelIds, PropertyReference propertiesReference) { if (!propertiesReference.isEmpty()) { var propertyValueIndex = (int) ((LongPropertyReference) propertiesReference).id; - var properties = batchNodeProperties.get(propertyValueIndex); - var importedProperties = new MutableInt(0); - properties.forEach((propertyKey, propertyValue) -> importedProperties.add(importProperty( - nodeReference, - propertyKey, - propertyValue - ))); - return importedProperties.intValue(); - } - return 0; - } + var properties = this.batchNodeProperties.get(propertyValueIndex); + + properties.forEach((propertyKey, propertyValue) -> { + var nodePropertyBuilder = this.threadLocalContext.nodePropertyBuilder(propertyKey); + assert nodePropertyBuilder != null : "observed property key that is not present in schema"; + nodePropertyBuilder.set(nodeReference, propertyValue); - private int importProperty(long neoNodeId, String propertyKey, Value value) { - var nodePropertyBuilder = this.threadLocalContext.nodePropertyBuilder(propertyKey); - if (nodePropertyBuilder != null) { - nodePropertyBuilder.set(neoNodeId, value); - return 1; + }); + return properties.size(); } return 0; } + + @Override + public void close() {} } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java index 0a85e24341f..7f05f025038 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilderContext.java @@ -117,12 +117,10 @@ private NodePropertiesFromStoreBuilder getPropertyBuilder(String propertyKey) { static class ThreadLocalContext { private static final long NOT_INITIALIZED = -42L; - private final long[] anyLabelArray = {NOT_INITIALIZED}; + private final long[] anyLabelArray = {NOT_INITIALIZED}; private final TokenToNodeLabels tokenToNodeLabels; - private final NodeLabelTokenToPropertyKeys nodeLabelTokenToPropertyKeys; - private final Function propertyBuilderFn; ThreadLocalContext( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java index 660aacf2e69..8dda9c87ed5 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/PropertyValues.java @@ -34,6 +34,8 @@ public abstract class PropertyValues { public abstract boolean isEmpty(); + public abstract int size(); + public abstract Iterable propertyKeys(); public static PropertyValues of(MapValue mapValue) { @@ -61,6 +63,11 @@ public boolean isEmpty() { return this.properties.isEmpty(); } + @Override + public int size() { + return this.properties.size(); + } + @Override public Set propertyKeys() { return this.properties.keySet(); @@ -88,6 +95,11 @@ public boolean isEmpty() { return this.properties.isEmpty(); } + @Override + public int size() { + return this.properties.size(); + } + @Override public Iterable propertyKeys() { return this.properties.keySet(); From cd5b61f3600f6bd61697e7c75e96adf035a3583f Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 31 Jan 2023 07:00:43 +0000 Subject: [PATCH 125/400] Update WCC sampling docs note Also enables the result assertion. --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index c320247adc6..7148f2feb94 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -626,12 +626,12 @@ CALL gds.graph.project( The following query is identical to the stream example in the xref:algorithms/wcc.adoc#algorithms-wcc-examples-seeding[previous section]. This time, we execute WCC on `myIndexedGraph` which will allow the algorithm to use the sampled strategy. -[role=query-example, no-result=true] +[role=query-example] -- .The following will run the algorithm with sampled strategy and stream results: [source, cypher, role=noplay] ---- -CALL gds.wcc.stream('myIndexedGraph') +CALL gds.wcc.stream('myIndexedGraph', { concurrency: 1 }) YIELD nodeId, componentId RETURN gds.util.asNode(nodeId).name AS name, componentId ORDER BY componentId, name @@ -652,6 +652,6 @@ ORDER BY componentId, name [NOTE] ==== -Because of the randomness in the Graph sampling optimization, running it another time might yield a different componentIds. +Because of the randomness in the Graph sampling optimization we are using `concurrency: 1` in the example, running it with higher concurrency may yield different componentIds but the actual components should stay the same. For our case here it would be equally plausible to get the inverse solution, f.i. when our community `0` nodes are mapped to community `3` instead, and vice versa. ==== From 9bad2b7db72d9360216133620c2e81d581aa387f Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 31 Jan 2023 08:40:12 +0000 Subject: [PATCH 126/400] Apply suggestions from code review Co-authored-by: jjaderberg --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index 7148f2feb94..de71cfc97c3 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -631,7 +631,7 @@ This time, we execute WCC on `myIndexedGraph` which will allow the algorithm to .The following will run the algorithm with sampled strategy and stream results: [source, cypher, role=noplay] ---- -CALL gds.wcc.stream('myIndexedGraph', { concurrency: 1 }) +CALL gds.wcc.stream('myIndexedGraph', {concurrency: 1}) YIELD nodeId, componentId RETURN gds.util.asNode(nodeId).name AS name, componentId ORDER BY componentId, name @@ -652,6 +652,7 @@ ORDER BY componentId, name [NOTE] ==== -Because of the randomness in the Graph sampling optimization we are using `concurrency: 1` in the example, running it with higher concurrency may yield different componentIds but the actual components should stay the same. +Because of the randomness in the Graph sampling optimization we are using `concurrency: 1` in the example. +Running it with higher concurrency may yield different componentIds but the actual components should stay the same. For our case here it would be equally plausible to get the inverse solution, f.i. when our community `0` nodes are mapped to community `3` instead, and vice versa. ==== From a03d28d9d1a3318640025579672aba41d3a40940 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 31 Jan 2023 12:31:06 +0100 Subject: [PATCH 127/400] Increase bounds to get the K1C flakey test safe Co-authored-by: Veselin Nikolov --- .../java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java b/algo/src/test/java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java index a3c233122c3..295eb7a5fb1 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/k1coloring/K1ColoringTest.java @@ -136,8 +136,8 @@ void testParallelK1Coloring() { .isLessThan(20L); assertThat(usedColors.size()) - .as("Used colors should be less than or equal to 20") - .isLessThanOrEqualTo(20); + .as("Used colors should be less than or equal to 21") + .isLessThanOrEqualTo(21); } From b18492b46d0a70f9f6296aedaddeb081d0fbe9a4 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Thu, 2 Feb 2023 10:06:18 +0100 Subject: [PATCH 128/400] Fix typo in HashGNN example docs --- .../node-embeddings/hashgnn.adoc | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc index 7f81c636c87..07c0d1a50de 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc @@ -354,7 +354,7 @@ CREATE (matt)-[:LIKES]->(mango), (jeff)-[:LIKES]->(mango), (brie)-[:LIKES]->(banana), - (else)-[:LIKES]->(plum), + (elsa)-[:LIKES]->(plum), (john)-[:LIKES]->(plum), (dan)-[:KNOWS]->(annie), @@ -574,17 +574,17 @@ YIELD nodeId, embedding .Results |=== | nodeId | embedding -| 0 | [1.0, 1.0, 0.0, 0.0, 0.0, 0.0] -| 1 | [1.0, 1.0, 0.0, 0.0, 0.0, 0.0] -| 2 | [1.0, 0.0, 1.0, 0.0, 0.0, 0.0] +| 0 | [1.0, 1.0, 0.0, 0.0, 0.0, 1.0] +| 1 | [1.0, 1.0, 1.0, 0.0, 0.0, 0.0] +| 2 | [1.0, 1.0, 1.0, 0.0, 0.0, 0.0] | 3 | [1.0, 0.0, 1.0, 0.0, 0.0, 0.0] -| 4 | [1.0, 1.0, 0.0, 0.0, 0.0, 0.0] -| 5 | [1.0, 1.0, 1.0, 0.0, 0.0, 0.0] +| 4 | [1.0, 1.0, 1.0, 0.0, 0.0, 0.0] +| 5 | [1.0, 1.0, 0.0, 0.0, 0.0, 0.0] | 6 | [1.0, 0.0, 0.0, 0.0, 0.0, 1.0] | 7 | [1.0, 0.0, 1.0, 0.0, 0.0, 0.0] | 8 | [1.0, 0.0, 0.0, 0.0, 0.0, 1.0] | 9 | [1.0, 0.0, 0.0, 0.0, 0.0, 1.0] -| 10 | [1.0, 0.0, 0.0, 0.0, 1.0, 0.0] +| 10 | [1.0, 0.0, 1.0, 0.0, 0.0, 0.0] |=== -- @@ -611,17 +611,17 @@ YIELD nodeId, embedding .Results |=== | nodeId | embedding -| 0 | [0.0, 0.8660253882408142, -0.8660253882408142, 0.0] -| 1 | [0.0, 0.8660253882408142, -0.8660253882408142, 0.0] -| 2 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] +| 0 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] +| 1 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] +| 2 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] | 3 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] -| 4 | [0.0, 0.8660253882408142, -0.8660253882408142, 0.0] -| 5 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] +| 4 | [0.0, 0.8660253882408142, -1.7320507764816284, 0.8660253882408142] +| 5 | [0.0, 0.8660253882408142, -0.8660253882408142, 0.0] | 6 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] | 7 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] | 8 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] | 9 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] -| 10 | [0.0, 0.0, -0.8660253882408142, 0.0] +| 10 | [0.0, 0.0, -1.7320507764816284, 0.8660253882408142] |=== -- From 6f65122b85bbeceb7911ddd305729f0a9d05fe80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 12:47:20 +0100 Subject: [PATCH 129/400] Make ElementSchema an interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../neo4j/gds/api/schema/ElementSchema.java | 106 +++++------------- 1 file changed, 29 insertions(+), 77 deletions(-) diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java index bff6c8a7062..4bbef10cd66 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java @@ -27,116 +27,68 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -public abstract class ElementSchema< - SELF extends ElementSchema, - ELEMENT_IDENTIFIER extends ElementIdentifier, - ENTRY extends ElementSchemaEntry, - PROPERTY_SCHEMA extends PropertySchema - > { +public interface ElementSchema< + SELF extends ElementSchema, + ELEMENT_IDENTIFIER extends ElementIdentifier, + ENTRY extends ElementSchemaEntry, + PROPERTY_SCHEMA extends PropertySchema> { - protected final Map entries; - ElementSchema(Map entries) { - this.entries = entries; - } + SELF filter(Set elementIdentifiersToKeep); - abstract SELF filter(Set elementIdentifiersToKeep); + SELF union(SELF other); - abstract SELF union(SELF other); + Collection entries(); - public Collection entries() { - return this.entries.values(); - } + ENTRY get(ELEMENT_IDENTIFIER identifier); - public void set(ENTRY entry) { - entries.put(entry.identifier(), entry); - } - public ENTRY get(ELEMENT_IDENTIFIER identifier) { - return entries.get(identifier); - } + Set allProperties(); - public void remove(ELEMENT_IDENTIFIER identifier) { - entries.remove(identifier); - } - - public Set allProperties() { - return this.entries - .values() + default Set allProperties(ELEMENT_IDENTIFIER elementIdentifier) { + return entries() .stream() .flatMap(entry -> entry.properties().keySet().stream()) .collect(Collectors.toSet()); } - public Set allProperties(ELEMENT_IDENTIFIER elementIdentifier) { - return Optional.ofNullable(this.entries.get(elementIdentifier)) - .map(entry -> entry.properties().keySet()) - .orElse(Set.of()); - } - - public boolean hasProperties() { - return entries.values().stream().anyMatch(entry -> !entry.properties().isEmpty()); + default boolean hasProperties() { + return entries().stream().anyMatch(entry -> !entry.properties().isEmpty()); } - public boolean hasProperty(ELEMENT_IDENTIFIER elementIdentifier, String propertyKey) { - return entries.containsKey(elementIdentifier) && entries - .get(elementIdentifier) - .properties() - .containsKey(propertyKey); + default boolean hasProperty(ELEMENT_IDENTIFIER elementIdentifier, String propertyKey) { + return Optional.ofNullable(get(elementIdentifier)) + .map(entry -> entry.properties().containsKey(propertyKey)) + .orElse(false); } - public List propertySchemasFor(ELEMENT_IDENTIFIER elementIdentifier) { + default List propertySchemasFor(ELEMENT_IDENTIFIER elementIdentifier) { return Optional - .ofNullable(entries.get(elementIdentifier)) + .ofNullable(get(elementIdentifier)) .map(entry -> entry.properties().values()) .map(ArrayList::new) .orElse(new ArrayList<>()); } - /** - * Returns a union of all properties in the given schema. - */ - public Map unionProperties() { - return entries - .values() + default Map unionProperties() { + return entries() .stream() .flatMap(e -> e.properties().entrySet().stream()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (leftSchema, rightSchema) -> leftSchema)); } - - public Map toMap() { - return entries - .entrySet() + default Map toMap() { + return entries() .stream() - .collect(Collectors.toMap(entry -> entry.getKey().name, entry -> entry.getValue().toMap())); + .collect(Collectors.toMap(entry -> entry.identifier().name, ElementSchemaEntry::toMap)); } - Map unionEntries(SELF other) { + default Map unionEntries(SELF other) { return Stream - .concat(entries.entrySet().stream(), other.entries.entrySet().stream()) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, ElementSchemaEntry::union)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - ElementSchema that = (ElementSchema) o; - - return entries.equals(that.entries); - } - - @Override - public int hashCode() { - return entries.hashCode(); - } - - @Override - public String toString() { - return toMap().toString(); + .concat(entries().stream(), other.entries().stream()) + .collect(Collectors.toMap(e -> e.identifier(), Function.identity(), ElementSchemaEntry::union)); } } From e8af2ee204b94b21e32275379a883047925fc27e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 12:47:39 +0100 Subject: [PATCH 130/400] Split NodeSchema interface and MutableNodeSchema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../gds/api/schema/MutableNodeSchema.java | 133 ++++++++++++++++++ .../org/neo4j/gds/api/schema/NodeSchema.java | 73 ++-------- 2 files changed, 142 insertions(+), 64 deletions(-) create mode 100644 graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java new file mode 100644 index 00000000000..898af1e52b5 --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public final class MutableNodeSchema implements NodeSchema { + + private final Map entries; + + public static MutableNodeSchema empty() { + return new MutableNodeSchema(new LinkedHashMap<>()); + } + + private MutableNodeSchema(Map entries) { + this.entries=entries; + } + + public static MutableNodeSchema from(NodeSchema fromSchema) { + var nodeSchema = MutableNodeSchema.empty(); + fromSchema.entries().forEach(fromEntry -> nodeSchema.set(MutableNodeSchemaEntry.from(fromEntry))); + + return nodeSchema; + } + + @Override + public Set availableLabels() { + return entries.keySet(); + } + + @Override + public boolean containsOnlyAllNodesLabel() { + return availableLabels().size() == 1 && availableLabels().contains(NodeLabel.ALL_NODES); + } + + @Override + public MutableNodeSchema filter(Set labelsToKeep) { + return new MutableNodeSchema(entries + .entrySet() + .stream() + .filter(e -> labelsToKeep.contains(e.getKey())) + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> MutableNodeSchemaEntry.from(entry.getValue()) + ))); + } + + @Override + public MutableNodeSchema union(NodeSchema other) { + return new MutableNodeSchema(unionEntries(other)); + } + + @Override + public Collection entries() { + return entries.values(); + } + + @Override + public MutableNodeSchemaEntry get(NodeLabel identifier) { + return entries.get(identifier); + } + + public void set(MutableNodeSchemaEntry entry) { + entries.put(entry.identifier(), entry); + } + public void remove(NodeLabel identifier) { + entries.remove(identifier); + } + + public MutableNodeSchemaEntry getOrCreateLabel(NodeLabel key) { + return this.entries.computeIfAbsent(key, (__) -> new MutableNodeSchemaEntry(key)); + } + + public MutableNodeSchema addLabel(NodeLabel nodeLabel) { + getOrCreateLabel(nodeLabel); + return this; + } + + public MutableNodeSchema addLabel(NodeLabel nodeLabel, Map nodeProperties) { + var nodeSchemaEntry = getOrCreateLabel(nodeLabel); + nodeProperties.forEach(nodeSchemaEntry::addProperty); + return this; + } + + public MutableNodeSchema addProperty(NodeLabel nodeLabel, String propertyName, PropertySchema propertySchema) { + getOrCreateLabel(nodeLabel).addProperty(propertyName, propertySchema); + return this; + } + + public MutableNodeSchema addProperty(NodeLabel nodeLabel, String propertyKey, ValueType valueType) { + getOrCreateLabel(nodeLabel).addProperty(propertyKey, valueType); + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MutableNodeSchema that = (MutableNodeSchema) o; + + return entries.equals(that.entries); + } + + @Override + public int hashCode() { + return entries.hashCode(); + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java index 712a3a84109..115cac7a049 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java @@ -20,77 +20,22 @@ package org.neo4j.gds.api.schema; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.api.nodeproperties.ValueType; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -public final class NodeSchema extends ElementSchema { +public interface NodeSchema extends ElementSchema { + Set availableLabels() ; - public static NodeSchema empty() { - return new NodeSchema(new LinkedHashMap<>()); - } - - private NodeSchema(Map entries) { - super(entries); - } - - public static NodeSchema from(NodeSchema fromSchema) { - var nodeSchema = NodeSchema.empty(); - fromSchema.entries().forEach(fromEntry -> nodeSchema.set(NodeSchemaEntry.from(fromEntry))); - - return nodeSchema; - } + boolean containsOnlyAllNodesLabel(); - public Set availableLabels() { - return entries.keySet(); - } - - public boolean containsOnlyAllNodesLabel() { - return availableLabels().size() == 1 && availableLabels().contains(NodeLabel.ALL_NODES); - } - - public NodeSchema filter(Set labelsToKeep) { - return new NodeSchema(entries - .entrySet() - .stream() - .filter(e -> labelsToKeep.contains(e.getKey())) - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> NodeSchemaEntry.from(entry.getValue()) - ))); - } + NodeSchema filter(Set labelsToKeep); @Override - public NodeSchema union(NodeSchema other) { - return new NodeSchema(unionEntries(other)); - } - - - public NodeSchemaEntry getOrCreateLabel(NodeLabel key) { - return this.entries.computeIfAbsent(key, (__) -> new NodeSchemaEntry(key)); - } - - public NodeSchema addLabel(NodeLabel nodeLabel) { - getOrCreateLabel(nodeLabel); - return this; - } - - public NodeSchema addLabel(NodeLabel nodeLabel, Map nodeProperties) { - var nodeSchemaEntry = getOrCreateLabel(nodeLabel); - nodeProperties.forEach(nodeSchemaEntry::addProperty); - return this; - } - - public NodeSchema addProperty(NodeLabel nodeLabel, String propertyName, PropertySchema propertySchema) { - getOrCreateLabel(nodeLabel).addProperty(propertyName, propertySchema); - return this; - } - - public NodeSchema addProperty(NodeLabel nodeLabel, String propertyKey, ValueType valueType) { - getOrCreateLabel(nodeLabel).addProperty(propertyKey, valueType); - return this; + default Set allProperties() { + return entries() + .stream() + .flatMap(entry -> entry.properties().keySet().stream()) + .collect(Collectors.toSet()); } } From a702624d239e3b75406b1f995710277bcadf6f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 12:47:54 +0100 Subject: [PATCH 131/400] Make ElementSchemaEntry an interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../gds/api/schema/ElementSchemaEntry.java | 49 +++---------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchemaEntry.java index 365fd97ef88..284a31388a7 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchemaEntry.java @@ -28,29 +28,17 @@ import static java.lang.String.format; -public abstract class ElementSchemaEntry, ELEMENT_IDENTIFIER extends ElementIdentifier, PROPERTY_SCHEMA extends PropertySchema> { +public interface ElementSchemaEntry, ELEMENT_IDENTIFIER extends ElementIdentifier, PROPERTY_SCHEMA extends PropertySchema> { - public final ELEMENT_IDENTIFIER identifier; - public final Map properties; + ELEMENT_IDENTIFIER identifier(); - public ElementSchemaEntry(ELEMENT_IDENTIFIER identifier, Map properties) { - this.identifier = identifier; - this.properties = properties; - } - - public ELEMENT_IDENTIFIER identifier() { - return identifier; - } + Map properties(); - public Map properties() { - return properties; - } + SELF union(SELF other); - abstract SELF union(SELF other); + Map toMap(); - abstract Map toMap(); - - protected Map unionProperties(Map rightProperties) { + default Map unionProperties(Map rightProperties) { return Stream .concat(properties().entrySet().stream(), rightProperties.entrySet().stream()) .collect(Collectors.toMap( @@ -58,7 +46,7 @@ protected Map unionProperties(Map entry.getValue().valueType())), @@ -72,27 +60,4 @@ protected Map unionProperties(Map that = (ElementSchemaEntry) o; - - if (!identifier.equals(that.identifier)) return false; - return properties.equals(that.properties); - } - - @Override - public int hashCode() { - int result = identifier.hashCode(); - result = 31 * result + properties.hashCode(); - return result; - } - - @Override - public String toString() { - return toMap().toString(); - } } From 94ffe3f7569e6f3d18b700009923389a704cb989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 12:48:12 +0100 Subject: [PATCH 132/400] Split NodeSchemaElement interface and MutableNodeSchemaElement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../api/schema/MutableNodeSchemaEntry.java | 127 ++++++++++++++++++ .../neo4j/gds/api/schema/NodeSchemaEntry.java | 71 +--------- 2 files changed, 128 insertions(+), 70 deletions(-) create mode 100644 graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchemaEntry.java diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchemaEntry.java new file mode 100644 index 00000000000..90a2b208bab --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchemaEntry.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public class MutableNodeSchemaEntry implements NodeSchemaEntry { + + private final NodeLabel nodeLabel; + private final Map properties; + + static MutableNodeSchemaEntry from(NodeSchemaEntry fromEntry) { + return new MutableNodeSchemaEntry(fromEntry.identifier(), new HashMap<>(fromEntry.properties())); + } + + MutableNodeSchemaEntry(NodeLabel identifier) { + this(identifier, new LinkedHashMap<>()); + } + + public MutableNodeSchemaEntry(NodeLabel nodeLabel, Map properties) { + this.nodeLabel = nodeLabel; + this.properties = properties; + } + + @Override + public NodeLabel identifier() { + return nodeLabel; + } + + @Override + public Map properties() { + return Map.copyOf(properties); + } + + @Override + public MutableNodeSchemaEntry union(MutableNodeSchemaEntry other) { + if (!other.identifier().equals(this.identifier())) { + throw new UnsupportedOperationException( + formatWithLocale( + "Cannot union node schema entries with different node labels %s and %s", + this.identifier(), + other.identifier() + ) + ); + } + + return new MutableNodeSchemaEntry(this.identifier(), unionProperties(other.properties)); + } + + @Override + public Map toMap() { + return properties + .entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) + ) + + ); + } + + public MutableNodeSchemaEntry addProperty(String propertyName, ValueType valueType) { + return addProperty( + propertyName, + PropertySchema.of( + propertyName, + valueType, + valueType.fallbackValue(), + PropertyState.PERSISTENT + ) + ); + } + + public MutableNodeSchemaEntry addProperty(String propertyName, PropertySchema propertySchema) { + this.properties.put(propertyName, propertySchema); + return this; + } + + public void removeProperty(String propertyName) { + properties.remove(propertyName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MutableNodeSchemaEntry that = (MutableNodeSchemaEntry) o; + + if (!nodeLabel.equals(that.nodeLabel)) return false; + return properties.equals(that.properties); + } + + @Override + public int hashCode() { + int result = nodeLabel.hashCode(); + result = 31 * result + properties.hashCode(); + return result; + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchemaEntry.java index cdd772d2c07..ffde9858669 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchemaEntry.java @@ -20,76 +20,7 @@ package org.neo4j.gds.api.schema; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.nodeproperties.ValueType; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.stream.Collectors; +public interface NodeSchemaEntry extends ElementSchemaEntry { -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; - -public class NodeSchemaEntry extends ElementSchemaEntry { - - NodeSchemaEntry(NodeLabel identifier) { - this(identifier, new LinkedHashMap<>()); - } - - public NodeSchemaEntry(NodeLabel identifier, Map properties) { - super(identifier, properties); - } - - static NodeSchemaEntry from(NodeSchemaEntry fromEntry) { - return new NodeSchemaEntry(fromEntry.identifier(), new HashMap<>(fromEntry.properties())); - } - - @Override - NodeSchemaEntry union(NodeSchemaEntry other) { - if (!other.identifier().equals(this.identifier())) { - throw new UnsupportedOperationException( - formatWithLocale( - "Cannot union node schema entries with different node labels %s and %s", - this.identifier(), - other.identifier() - ) - ); - } - - return new NodeSchemaEntry(this.identifier(), unionProperties(other.properties)); - } - - public NodeSchemaEntry addProperty(String propertyName, ValueType valueType) { - return addProperty( - propertyName, - PropertySchema.of( - propertyName, - valueType, - valueType.fallbackValue(), - PropertyState.PERSISTENT - ) - ); - } - - public NodeSchemaEntry addProperty(String propertyName, PropertySchema propertySchema) { - this.properties.put(propertyName, propertySchema); - return this; - } - - public void removeProperty(String propertyName) { - properties.remove(propertyName); - } - - @Override - Map toMap() { - return properties - .entrySet() - .stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) - ) - - ); - } } From 6974c61e55a8ae094175b4adc0ba6bdd39afc8e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 13:00:26 +0100 Subject: [PATCH 133/400] Split RelationshipSchema interface and MutableRelationshipSchema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../api/schema/MutableRelationshipSchema.java | 159 ++++++++++++++++++ .../gds/api/schema/RelationshipSchema.java | 109 +----------- 2 files changed, 165 insertions(+), 103 deletions(-) create mode 100644 graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java new file mode 100644 index 00000000000..a68ae37b7b0 --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.nodeproperties.ValueType; + +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class MutableRelationshipSchema implements RelationshipSchema { + + private final Map entries; + + public static MutableRelationshipSchema empty() { + return new MutableRelationshipSchema(new LinkedHashMap<>()); + } + + public MutableRelationshipSchema(Map entries) { + this.entries = entries; + } + + public static MutableRelationshipSchema from(RelationshipSchema fromSchema) { + var relationshipSchema = MutableRelationshipSchema.empty(); + fromSchema.entries().forEach(fromEntry -> relationshipSchema.set(MutableRelationshipSchemaEntry.from(fromEntry))); + + return relationshipSchema; + } + + @Override + public MutableRelationshipSchema filter(Set relationshipTypesToKeep) { + return new MutableRelationshipSchema(entries + .entrySet() + .stream() + .filter(e -> relationshipTypesToKeep.contains(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, entry -> MutableRelationshipSchemaEntry.from(entry.getValue())))); + } + + @Override + public MutableRelationshipSchema union(MutableRelationshipSchema other) { + return new MutableRelationshipSchema(unionEntries(other)); + } + + @Override + public Collection entries() { + return entries.values(); + } + + @Override + public MutableRelationshipSchemaEntry get(RelationshipType identifier) { + return entries.get(identifier); + } + + @Override + public Set availableTypes() { + return entries.keySet(); + } + + @Override + public boolean isUndirected() { + // a graph with no relationships is considered undirected + // this is because algorithms such as TriangleCount are still well-defined + // so it is the least restrictive decision + return entries.values().stream().allMatch(MutableRelationshipSchemaEntry::isUndirected); + } + + @Override + public boolean isUndirected(RelationshipType type) { + return entries.get(type).isUndirected(); + } + + // TODO: remove + @Override + public Map directions() { + return availableTypes() + .stream() + .collect(Collectors.toMap( + relationshipType -> relationshipType, + relationshipType -> entries.get(relationshipType).direction() + )); + } + + @Override + public Object toMapOld() { + return entries() + .stream() + .collect(Collectors.toMap(e -> e.identifier().name(), + e -> e + .properties() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, + innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) + )) + )); + } + + public void set(MutableRelationshipSchemaEntry entry) { + entries.put(entry.identifier(), entry); + } + public void remove(RelationshipType identifier) { + entries.remove(identifier); + } + + public MutableRelationshipSchemaEntry getOrCreateRelationshipType( + RelationshipType relationshipType, Direction direction + ) { + return this.entries.computeIfAbsent(relationshipType, + __ -> new MutableRelationshipSchemaEntry(relationshipType, direction) + ); + } + + public MutableRelationshipSchema addRelationshipType(RelationshipType relationshipType, Direction direction) { + getOrCreateRelationshipType(relationshipType, direction); + return this; + } + + public MutableRelationshipSchema addProperty( + RelationshipType relationshipType, + Direction direction, + String propertyKey, + RelationshipPropertySchema propertySchema + ) { + getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, propertySchema); + return this; + } + + public MutableRelationshipSchema addProperty( + RelationshipType relationshipType, + Direction direction, + String propertyKey, + ValueType valueType, + PropertyState propertyState + ) { + getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, valueType, propertyState); + return this; + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java index a82f26ddc80..bc403682719 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java @@ -20,116 +20,19 @@ package org.neo4j.gds.api.schema; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.nodeproperties.ValueType; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; +public interface RelationshipSchema extends ElementSchema { + Set availableTypes(); -public class RelationshipSchema extends ElementSchema { + boolean isUndirected(); - public static RelationshipSchema empty() { - return new RelationshipSchema(new LinkedHashMap<>()); - } - - public RelationshipSchema(Map entries) { - super(entries); - } - - public static RelationshipSchema from(RelationshipSchema fromSchema) { - var relationshipSchema = RelationshipSchema.empty(); - fromSchema.entries().forEach(fromEntry -> relationshipSchema.set(RelationshipSchemaEntry.from(fromEntry))); - - return relationshipSchema; - } - - @Override - public RelationshipSchema filter(Set relationshipTypesToKeep) { - return new RelationshipSchema(entries - .entrySet() - .stream() - .filter(e -> relationshipTypesToKeep.contains(e.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, entry -> RelationshipSchemaEntry.from(entry.getValue())))); - } - - @Override - public RelationshipSchema union(RelationshipSchema other) { - return new RelationshipSchema(unionEntries(other)); - } - - public Set availableTypes() { - return entries.keySet(); - } - - public boolean isUndirected() { - // a graph with no relationships is considered undirected - // this is because algorithms such as TriangleCount are still well-defined - // so it is the least restrictive decision - return entries.values().stream().allMatch(RelationshipSchemaEntry::isUndirected); - } - - public boolean isUndirected(RelationshipType type) { - return entries.get(type).isUndirected(); - } - - public RelationshipSchemaEntry getOrCreateRelationshipType( - RelationshipType relationshipType, Direction direction - ) { - return this.entries.computeIfAbsent(relationshipType, - __ -> new RelationshipSchemaEntry(relationshipType, direction) - ); - } - - public RelationshipSchema addRelationshipType(RelationshipType relationshipType, Direction direction) { - getOrCreateRelationshipType(relationshipType, direction); - return this; - } - - public RelationshipSchema addProperty( - RelationshipType relationshipType, - Direction direction, - String propertyKey, - RelationshipPropertySchema propertySchema - ) { - getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, propertySchema); - return this; - } - - public RelationshipSchema addProperty( - RelationshipType relationshipType, - Direction direction, - String propertyKey, - ValueType valueType, - PropertyState propertyState - ) { - getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, valueType, propertyState); - return this; - } + boolean isUndirected(RelationshipType type); // TODO: remove - public Map directions() { - return availableTypes() - .stream() - .collect(Collectors.toMap( - relationshipType -> relationshipType, - relationshipType -> entries.get(relationshipType).direction() - )); - } + Map directions(); - Object toMapOld() { - return entries() - .stream() - .collect(Collectors.toMap(e -> e.identifier().name(), - e -> e - .properties() - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, - innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) - )) - )); - } + Object toMapOld(); } From 60fd038172515f29e28ecdc6701098682939445b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 13:00:45 +0100 Subject: [PATCH 134/400] Split RelationshipSchemaEntry interface and MutableRelationshipSchemaEntry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../MutableRelationshipSchemaEntry.java | 134 ++++++++++++++++++ .../api/schema/RelationshipSchemaEntry.java | 115 +-------------- 2 files changed, 137 insertions(+), 112 deletions(-) create mode 100644 graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java new file mode 100644 index 00000000000..cd479283748 --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.PropertyState; +import org.neo4j.gds.api.nodeproperties.ValueType; +import org.neo4j.gds.core.Aggregation; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public class MutableRelationshipSchemaEntry implements RelationshipSchemaEntry { + + private final RelationshipType relationshipType; + private final Direction direction; + private final Map properties; + + MutableRelationshipSchemaEntry(RelationshipType identifier, Direction direction) { + this(identifier, direction, new LinkedHashMap<>()); + } + + public MutableRelationshipSchemaEntry( + RelationshipType relationshipType, + Direction direction, + Map properties + ) { + this.relationshipType = relationshipType; + this.direction = direction; + this.properties = properties; + } + + static MutableRelationshipSchemaEntry from(RelationshipSchemaEntry fromEntry) { + return new MutableRelationshipSchemaEntry( + fromEntry.identifier(), + fromEntry.direction(), + new HashMap<>(fromEntry.properties()) + ); + } + + @Override + public Direction direction() { + return this.direction; + } + + @Override + public boolean isUndirected() { + return direction() == Direction.UNDIRECTED; + } + + @Override + public RelationshipType identifier() { + return relationshipType; + } + + @Override + public Map properties() { + return Map.copyOf(properties); + } + + @Override + public MutableRelationshipSchemaEntry union(MutableRelationshipSchemaEntry other) { + if (!other.identifier().equals(this.identifier())) { + throw new UnsupportedOperationException( + formatWithLocale( + "Cannot union relationship schema entries with different types %s and %s", + this.identifier(), + other.identifier() + ) + ); + } + + if (other.isUndirected() != this.isUndirected()) { + throw new IllegalArgumentException( + formatWithLocale("Conflicting directionality for relationship types %s", this.identifier().name)); + } + + return new MutableRelationshipSchemaEntry(this.identifier(), this.direction, unionProperties(other.properties)); + } + + @Override + public Map toMap() { + return Map.of( + "direction", direction.name(), + "properties", properties + .entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) + ) + ) + ); + } + + public MutableRelationshipSchemaEntry addProperty(String propertyKey, ValueType valueType, PropertyState propertyState) { + return addProperty( + propertyKey, + RelationshipPropertySchema.of( + propertyKey, + valueType, + valueType.fallbackValue(), + propertyState, + Aggregation.DEFAULT + ) + ); + } + + public MutableRelationshipSchemaEntry addProperty(String propertyKey, RelationshipPropertySchema propertySchema) { + this.properties.put(propertyKey, propertySchema); + return this; + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java index d468a96b722..0f70faeec82 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java @@ -20,118 +20,9 @@ package org.neo4j.gds.api.schema; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.core.Aggregation; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.stream.Collectors; +public interface RelationshipSchemaEntry extends ElementSchemaEntry { + Direction direction(); -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; - -public class RelationshipSchemaEntry extends ElementSchemaEntry { - - private final Direction direction; - - RelationshipSchemaEntry(RelationshipType identifier, Direction direction) { - this(identifier, direction, new LinkedHashMap<>()); - } - - public RelationshipSchemaEntry( - RelationshipType identifier, - Direction direction, - Map properties - ) { - super(identifier, properties); - this.direction = direction; - } - - static RelationshipSchemaEntry from(RelationshipSchemaEntry fromEntry) { - return new RelationshipSchemaEntry( - fromEntry.identifier(), - fromEntry.direction, - new HashMap<>(fromEntry.properties()) - ); - } - - public Direction direction() { - return this.direction; - } - - boolean isUndirected() { - return direction() == Direction.UNDIRECTED; - } - - @Override - RelationshipSchemaEntry union(RelationshipSchemaEntry other) { - if (!other.identifier().equals(this.identifier())) { - throw new UnsupportedOperationException( - formatWithLocale( - "Cannot union relationship schema entries with different types %s and %s", - this.identifier(), - other.identifier() - ) - ); - } - - if (other.isUndirected() != this.isUndirected()) { - throw new IllegalArgumentException( - formatWithLocale("Conflicting directionality for relationship types %s", this.identifier().name)); - } - - return new RelationshipSchemaEntry(this.identifier(), this.direction, unionProperties(other.properties)); - } - - public RelationshipSchemaEntry addProperty(String propertyKey, ValueType valueType, PropertyState propertyState) { - return addProperty( - propertyKey, - RelationshipPropertySchema.of( - propertyKey, - valueType, - valueType.fallbackValue(), - propertyState, - Aggregation.DEFAULT - ) - ); - } - - public RelationshipSchemaEntry addProperty(String propertyKey, RelationshipPropertySchema propertySchema) { - this.properties.put(propertyKey, propertySchema); - return this; - } - - @Override - Map toMap() { - return Map.of( - "direction", direction.name(), - "properties", properties - .entrySet() - .stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - innerEntry -> GraphSchema.forPropertySchema(innerEntry.getValue()) - ) - ) - ); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - RelationshipSchemaEntry that = (RelationshipSchemaEntry) o; - - return direction == that.direction; - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + direction.hashCode(); - return result; - } + boolean isUndirected(); } From ef93867e6b3cd7b24550f4869c40bec4366e93d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 13:01:11 +0100 Subject: [PATCH 135/400] Move implementation of ElementSchema.allProperties into interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../java/org/neo4j/gds/api/schema/ElementSchema.java | 7 ++++++- .../main/java/org/neo4j/gds/api/schema/NodeSchema.java | 9 --------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java index 4bbef10cd66..311e2f0fafe 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java @@ -46,7 +46,12 @@ public interface ElementSchema< ENTRY get(ELEMENT_IDENTIFIER identifier); - Set allProperties(); + default Set allProperties() { + return entries() + .stream() + .flatMap(entry -> entry.properties().keySet().stream()) + .collect(Collectors.toSet()); + } default Set allProperties(ELEMENT_IDENTIFIER elementIdentifier) { return entries() diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java index 115cac7a049..c98fc9468d6 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java @@ -22,7 +22,6 @@ import org.neo4j.gds.NodeLabel; import java.util.Set; -import java.util.stream.Collectors; public interface NodeSchema extends ElementSchema { Set availableLabels() ; @@ -30,12 +29,4 @@ public interface NodeSchema extends ElementSchema labelsToKeep); - - @Override - default Set allProperties() { - return entries() - .stream() - .flatMap(entry -> entry.properties().keySet().stream()) - .collect(Collectors.toSet()); - } } From 9ee523cc5861338a665abf156674c1db12bb3760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 15:24:01 +0100 Subject: [PATCH 136/400] Split GraphSchema and MutableGraphSchema --- .../embeddings/graphsage/GraphSageHelper.java | 3 +- .../algo/GraphSageAlgorithmFactoryTest.java | 10 +- .../neo4j/gds/api/CSRGraphStoreFactory.java | 4 +- .../neo4j/gds/core/loading/CypherFactory.java | 4 +- .../neo4j/gds/core/loading/NativeFactory.java | 4 +- .../org/neo4j/gds/core/loading/Nodes.java | 3 +- .../gds/projection/CypherAggregationTest.java | 2 +- .../neo4j/gds/doc/ModelCatalogDocTest.java | 4 +- .../neo4j/gds/api/schema/ElementSchema.java | 2 +- .../org/neo4j/gds/api/schema/GraphSchema.java | 66 ++--------- .../gds/api/schema/MutableGraphSchema.java | 111 ++++++++++++++++++ .../api/schema/MutableRelationshipSchema.java | 2 +- .../gds/api/schema/RelationshipSchema.java | 2 +- .../GraphStoreRelationshipVisitorTest.java | 3 +- .../LinkPredictionTrainingPipelineTest.java | 6 +- .../graphsage/GraphSageTrainProcTest.java | 4 +- .../LinkPredictionPipelineProcTestBase.java | 4 +- ...PredictionPredictPipelineExecutorTest.java | 4 +- ...sificationPipelinePredictProcTestUtil.java | 4 +- ...sificationPredictPipelineExecutorTest.java | 4 +- .../predict/NodeRegressionModelTestUtil.java | 4 +- 21 files changed, 159 insertions(+), 91 deletions(-) create mode 100644 graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageHelper.java b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageHelper.java index 9c80628b99c..25f2f3e6d1d 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageHelper.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageHelper.java @@ -24,6 +24,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.NodeSchemaEntry; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.mem.MemoryEstimations; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -278,7 +279,7 @@ private static Map> propertyKeysPerNodeLabel(GraphSchema .nodeSchema() .entries() .stream() - .collect(Collectors.toMap(e -> e.identifier, e -> e.properties().keySet())); + .collect(Collectors.toMap(NodeSchemaEntry::identifier, e -> e.properties().keySet())); } private static Map> filteredPropertyKeysPerNodeLabel(Graph graph, GraphSageTrainConfig config) { diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactoryTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactoryTest.java index 1b065966c3a..546f749c02a 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactoryTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactoryTest.java @@ -27,7 +27,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.model.InjectModelCatalog; @@ -279,7 +279,7 @@ void memoryEstimationTreeStructure() { var model = Model.of( "graphSage", - GraphSchema.empty(), + MutableGraphSchema.empty(), ModelData.of(new Layer[]{}, new SingleLabelFeatureFunction()), trainConfig, GraphSageModelTrainer.GraphSageTrainMetrics.empty() @@ -335,7 +335,7 @@ void memoryEstimationMutateTreeStructure() { var model = Model.of( "graphSage", - GraphSchema.empty(), + MutableGraphSchema.empty(), ModelData.of(new Layer[]{}, new SingleLabelFeatureFunction()), trainConfig, GraphSageModelTrainer.GraphSageTrainMetrics.empty() @@ -508,7 +508,7 @@ public String toString() { var model = Model.of( "graphSage", - GraphSchema.empty(), + MutableGraphSchema.empty(), ModelData.of(new Layer[]{}, new SingleLabelFeatureFunction()), trainConfig, GraphSageModelTrainer.GraphSageTrainMetrics.empty() @@ -555,7 +555,7 @@ void mutateHasPersistentPart() { var model = Model.of( "graphSage", - GraphSchema.empty(), + MutableGraphSchema.empty(), ModelData.of(new Layer[]{}, new SingleLabelFeatureFunction()), trainConfig, GraphSageModelTrainer.GraphSageTrainMetrics.empty() diff --git a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java index ec4aac447ec..aec5bda1535 100644 --- a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.api; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.loading.CSRGraphStore; @@ -61,7 +61,7 @@ protected void logLoadingSummary(GraphStore graphStore) { } } - protected abstract GraphSchema computeGraphSchema( + protected abstract MutableGraphSchema computeGraphSchema( Nodes nodes, RelationshipImportResult relationshipImportResult ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index 4653d96dcc6..de26cc597c7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -32,7 +32,7 @@ import org.neo4j.gds.api.CSRGraphStoreFactory; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphLoaderContext; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.config.GraphProjectFromCypherConfig; @@ -119,7 +119,7 @@ public GraphDimensions estimationDimensions() { } @Override - protected GraphSchema computeGraphSchema( + protected MutableGraphSchema computeGraphSchema( Nodes nodes, RelationshipImportResult relationshipImportResult ) { return CSRGraphStoreUtil.computeGraphSchema( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java index 89df81b651d..f82975a1d2a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java @@ -28,7 +28,7 @@ import org.neo4j.gds.api.CSRGraphStoreFactory; import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.GraphDimensions; @@ -296,7 +296,7 @@ protected ProgressTracker initProgressTracker() { } @Override - protected GraphSchema computeGraphSchema( + protected MutableGraphSchema computeGraphSchema( Nodes nodes, RelationshipImportResult relationshipImportResult ) { return CSRGraphStoreUtil.computeGraphSchema( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index c56be2ebc1c..8aee754ca22 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -30,6 +30,7 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.ImmutablePropertySchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.NodeSchema; import java.util.Map; @@ -52,7 +53,7 @@ static Nodes of( Map propertyValues, PropertyState propertyState ) { - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); var nodePropertyStoreBuilder = NodePropertyStore.builder(); propertyMappings.forEach(((nodeLabel, mappings) -> { diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index aff73b20f28..0ff2bd02465 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -684,7 +684,7 @@ void testRespectUndirectedTypes(List undirectedConfig, List expe .relationshipSchema() .entries() .forEach(entry -> { - var expectedDirection = expectedUndirectedTypes.contains(entry.identifier.name) ? org.neo4j.gds.api.schema.Direction.UNDIRECTED : org.neo4j.gds.api.schema.Direction.DIRECTED; + var expectedDirection = expectedUndirectedTypes.contains(entry.identifier().name) ? org.neo4j.gds.api.schema.Direction.UNDIRECTED : org.neo4j.gds.api.schema.Direction.DIRECTED; assertThat(entry.direction()).isEqualTo(expectedDirection); }); } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/ModelCatalogDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/ModelCatalogDocTest.java index 01579b8b68b..8a29d3b80dc 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/ModelCatalogDocTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/ModelCatalogDocTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.embeddings.graphsage.GraphSageModelTrainer; @@ -43,7 +43,7 @@ abstract class ModelCatalogDocTest extends SingleFileDocTestBase { void loadModel() { modelCatalog.set(Model.of( GraphSage.MODEL_TYPE, - GraphSchema.empty(), + MutableGraphSchema.empty(), ModelData.of(new Layer[0], new SingleLabelFeatureFunction()), ImmutableGraphSageTrainConfig .builder() diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java index 311e2f0fafe..e96981be27e 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java @@ -94,6 +94,6 @@ default Map toMap() { default Map unionEntries(SELF other) { return Stream .concat(entries().stream(), other.entries().stream()) - .collect(Collectors.toMap(e -> e.identifier(), Function.identity(), ElementSchemaEntry::union)); + .collect(Collectors.toMap(ElementSchemaEntry::identifier, Function.identity(), ElementSchemaEntry::union)); } } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java index 804cabc2dc9..b1bf9601faf 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java @@ -21,15 +21,12 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.annotation.ValueClass; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; -@ValueClass public interface GraphSchema { NodeSchema nodeSchema(); @@ -38,6 +35,12 @@ public interface GraphSchema { Map graphProperties(); + GraphSchema filterNodeLabels(Set labelsToKeep); + + GraphSchema filterRelationshipTypes(Set relationshipTypesToKeep); + + GraphSchema union(GraphSchema other); + default Map toMap() { return Map.of( "nodes", nodeSchema().toMap(), @@ -60,50 +63,12 @@ default Map toMapOld() { ); } - default GraphSchema filterNodeLabels(Set labelsToKeep) { - return of(nodeSchema().filter(labelsToKeep), relationshipSchema(), graphProperties()); - } - - default GraphSchema filterRelationshipTypes(Set relationshipTypesToKeep) { - return of(nodeSchema(), relationshipSchema().filter(relationshipTypesToKeep), graphProperties()); - } - - default GraphSchema union(GraphSchema other) { - return GraphSchema.of( - nodeSchema().union(other.nodeSchema()), - relationshipSchema().union(other.relationshipSchema()), - unionGraphProperties(other.graphProperties()) - ); - } - - private Map unionGraphProperties(Map otherProperties) { - return Stream.concat( - graphProperties().entrySet().stream(), - otherProperties.entrySet().stream() - ).collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue, - (leftType, rightType) -> { - if (leftType.valueType() != rightType.valueType()) { - throw new IllegalArgumentException(String.format( - Locale.ENGLISH, - "Combining schema entries with value type %s and %s is not supported.", - leftType.valueType(), - rightType.valueType() - )); - } else { - return leftType; - } - } - )); + default boolean isUndirected() { + return relationshipSchema().isUndirected(); } - static GraphSchema of(NodeSchema nodeSchema, RelationshipSchema relationshipSchema, Map graphProperties) { - return ImmutableGraphSchema.builder() - .nodeSchema(nodeSchema) - .relationshipSchema(relationshipSchema) - .graphProperties(graphProperties) - .build(); + default Direction direction() { + return relationshipSchema().isUndirected() ? Direction.UNDIRECTED : Direction.DIRECTED; } static String forPropertySchema(PS propertySchema) { @@ -125,15 +90,4 @@ static String forPropertySchema(PS propertySchema) { ); } - static GraphSchema empty() { - return of(NodeSchema.empty(), RelationshipSchema.empty(), Map.of()); - } - - default boolean isUndirected() { - return relationshipSchema().isUndirected(); - } - - default Direction direction() { - return relationshipSchema().isUndirected() ? Direction.UNDIRECTED : Direction.DIRECTED; - } } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java new file mode 100644 index 00000000000..88dec2644ec --- /dev/null +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.api.schema; + +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.annotation.ValueClass; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@ValueClass +public +interface MutableGraphSchema extends GraphSchema { + @Override + MutableNodeSchema nodeSchema(); + + @Override + MutableRelationshipSchema relationshipSchema(); + + + @Override + default MutableGraphSchema filterNodeLabels(Set labelsToKeep) { + return of(nodeSchema().filter(labelsToKeep), relationshipSchema(), graphProperties()); + } + + @Override + default MutableGraphSchema filterRelationshipTypes(Set relationshipTypesToKeep) { + return of(nodeSchema(), relationshipSchema().filter(relationshipTypesToKeep), graphProperties()); + } + + @Override + default MutableGraphSchema union(GraphSchema other) { + return MutableGraphSchema.of( + nodeSchema().union(other.nodeSchema()), + relationshipSchema().union(other.relationshipSchema()), + unionGraphProperties(other.graphProperties()) + ); + } + + static MutableGraphSchema empty() { + return of(MutableNodeSchema.empty(), MutableRelationshipSchema.empty(), Map.of()); + } + + static MutableGraphSchema from(GraphSchema from) { + return of( + MutableNodeSchema.from(from.nodeSchema()), + MutableRelationshipSchema.from(from.relationshipSchema()), + new HashMap<>(from.graphProperties()) + ); + } + + static MutableGraphSchema of( + MutableNodeSchema nodeSchema, + MutableRelationshipSchema relationshipSchema, + Map graphProperties + ) { + return ImmutableMutableGraphSchema.builder() + .nodeSchema(nodeSchema) + .relationshipSchema(relationshipSchema) + .graphProperties(graphProperties) + .build(); + } + + static ImmutableMutableGraphSchema.Builder builder() { + return ImmutableMutableGraphSchema.builder(); + } + + private Map unionGraphProperties(Map otherProperties) { + return Stream.concat( + graphProperties().entrySet().stream(), + otherProperties.entrySet().stream() + ).collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (leftType, rightType) -> { + if (leftType.valueType() != rightType.valueType()) { + throw new IllegalArgumentException(String.format( + Locale.ENGLISH, + "Combining schema entries with value type %s and %s is not supported.", + leftType.valueType(), + rightType.valueType() + )); + } else { + return leftType; + } + } + )); + } +} diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java index a68ae37b7b0..b04ace26103 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java @@ -58,7 +58,7 @@ public MutableRelationshipSchema filter(Set relationshipTypesT } @Override - public MutableRelationshipSchema union(MutableRelationshipSchema other) { + public MutableRelationshipSchema union(RelationshipSchema other) { return new MutableRelationshipSchema(unionEntries(other)); } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java index bc403682719..4c1d3e56416 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchema.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.Set; -public interface RelationshipSchema extends ElementSchema { +public interface RelationshipSchema extends ElementSchema { Set availableTypes(); boolean isUndirected(); diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java index a7f5363c28a..97abfd9b202 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java @@ -25,6 +25,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.RelationshipPropertyStore; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.ImmutableNodes; @@ -158,7 +159,7 @@ private Graph createGraph( Map propertyStores = actualRelationships.properties(); return new GraphStoreBuilder() - .schema(expectedGraph.schema()) + .schema(MutableGraphSchema.from(expectedGraph.schema())) .capabilities(ImmutableStaticCapabilities.of(true)) .nodes(nodes) .relationshipImportResult(RelationshipImportResult.of( diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java index 2fabcd53096..79443714c44 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.model.OpenModelCatalog; @@ -191,7 +191,7 @@ void deriveRelationshipWeightPropertyFromTrainedModel() { String modelName = "myModel"; modelCatalog.set(Model.of( "myAlgo", - GraphSchema.empty(), + MutableGraphSchema.empty(), 1L, TestWeightedTrainConfigImpl.builder() .modelUser("") @@ -223,7 +223,7 @@ void notDerivePropertyFromUnweightedTrainedModel() { String modelName = "myModel"; modelCatalog.set(Model.of( "myAlgo", - GraphSchema.empty(), + MutableGraphSchema.empty(), 1L, TestTrainConfigImpl.builder() .modelUser("") diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java index f4251641d00..d80a8e590d8 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java @@ -27,7 +27,7 @@ import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.TestProcedureRunner; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.loading.CSRGraphStore; import org.neo4j.gds.core.loading.GraphStoreCatalog; @@ -233,7 +233,7 @@ void shouldValidateModelBeforeTraining() { ); var model = Model.of( GraphSage.MODEL_TYPE, - GraphSchema.empty(), + MutableGraphSchema.empty(), 42, config, GraphSageModelTrainer.GraphSageTrainMetrics.empty() diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineProcTestBase.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineProcTestBase.java index 8b3837802d1..fa7f4e2fee5 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineProcTestBase.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineProcTestBase.java @@ -25,7 +25,7 @@ import org.neo4j.gds.GdsCypher; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.DefaultValue; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.catalog.GraphListProc; import org.neo4j.gds.catalog.GraphProjectProc; import org.neo4j.gds.core.model.Model; @@ -103,7 +103,7 @@ private void withModelInCatalog() { modelCatalog.set(Model.of( MODEL_TYPE, - GraphSchema.empty(), + MutableGraphSchema.empty(), modelData, LinkPredictionTrainConfigImpl.builder() .modelUser(getUsername()) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java index 84108b351ac..910189fd3f0 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java @@ -26,7 +26,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; @@ -355,7 +355,7 @@ void progressTracking() { Model.of( MODEL_TYPE, - GraphSchema.empty(), + MutableGraphSchema.empty(), modelData, LinkPredictionTrainConfigImpl.builder() .modelUser(username) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelinePredictProcTestUtil.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelinePredictProcTestUtil.java index e45fab90736..2b3d62791de 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelinePredictProcTestUtil.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelinePredictProcTestUtil.java @@ -21,7 +21,7 @@ import org.apache.commons.lang3.mutable.MutableDouble; import org.junit.jupiter.params.provider.Arguments; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -100,7 +100,7 @@ static Model model = Model.of( NodeClassificationTrainingPipeline.MODEL_TYPE, - GraphSchema.empty(), + MutableGraphSchema.empty(), modelData, NodeClassificationPipelineTrainConfigImpl.builder() .modelUser(getUsername()) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionModelTestUtil.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionModelTestUtil.java index 1d9d74f4b80..d477cc1a393 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionModelTestUtil.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionModelTestUtil.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.ml.pipeline.node.regression.predict; -import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.ml.core.functions.Weights; import org.neo4j.gds.ml.core.tensor.Matrix; @@ -53,7 +53,7 @@ static Model Date: Tue, 24 Jan 2023 15:24:53 +0100 Subject: [PATCH 137/400] Adapt code base to schema interface split --- .../java/org/neo4j/gds/mem/HugeArrays.java | 2 +- .../beta/generator/RandomGraphGenerator.java | 12 ++--- .../neo4j/gds/core/loading/CSRGraphStore.java | 33 ++++++------ .../gds/core/loading/CSRGraphStoreUtil.java | 16 +++--- .../gds/core/loading/LazyIdMapBuilder.java | 4 +- .../org/neo4j/gds/core/loading/Nodes.java | 3 +- .../core/loading/SingleTypeRelationships.java | 10 ++-- .../loading/construction/GraphFactory.java | 6 ++- .../loading/construction/NodesBuilder.java | 6 +-- .../generator/RandomGraphGeneratorTest.java | 40 +++++++------- .../org/neo4j/gds/config/WriteConfigTest.java | 8 +-- .../NodeLabelTokenToPropertyKeysTest.java | 22 +++++--- .../neo4j/gds/projection/GraphAggregator.java | 15 +++--- .../neo4j/gds/api/schema/GraphSchemaTest.java | 42 +++++++-------- .../neo4j/gds/api/schema/NodeSchemaTest.java | 38 +++++++------- .../api/schema/RelationshipSchemaTest.java | 52 ++++++++++--------- .../org/neo4j/gds/core/io/file/FileInput.java | 8 +-- .../io/file/FileToGraphStoreImporter.java | 10 ++-- .../io/schema/NodeSchemaBuilderVisitor.java | 8 +-- .../RelationshipSchemaBuilderVisitor.java | 8 +-- .../io/file/GraphStoreNodeVisitorTest.java | 12 ++--- .../GraphStoreRelationshipVisitorTest.java | 3 +- .../schema/NodeSchemaBuilderVisitorTest.java | 6 +-- .../gds/core/io/file/NodeFileHeader.java | 6 +-- .../core/io/file/RelationshipFileHeader.java | 6 +-- .../gds/core/io/file/csv/CsvFileInput.java | 28 +++++----- .../io/file/csv/GraphStoreToCsvExporter.java | 4 +- .../core/io/file/csv/NodeSchemaLoader.java | 4 +- .../io/file/csv/RelationshipSchemaLoader.java | 4 +- .../core/io/file/csv/CsvNodeVisitorTest.java | 10 ++-- .../file/csv/CsvRelationshipVisitorTest.java | 6 +-- .../io/file/csv/NodeSchemaLoaderTest.java | 10 ++-- .../csv/RelationshipSchemaLoaderTest.java | 14 ++--- ...odeClassificationToModelConverterTest.java | 15 +++--- .../gds/beta/filter/GraphStoreFilter.java | 18 ++++--- .../java/org/neo4j/gds/gdl/GdlFactory.java | 14 ++--- .../org/neo4j/gds/gdl/GdlFactoryTest.java | 10 ++-- 37 files changed, 267 insertions(+), 246 deletions(-) diff --git a/collections/src/main/java/org/neo4j/gds/mem/HugeArrays.java b/collections/src/main/java/org/neo4j/gds/mem/HugeArrays.java index a792c21684e..8f2bc3f6706 100644 --- a/collections/src/main/java/org/neo4j/gds/mem/HugeArrays.java +++ b/collections/src/main/java/org/neo4j/gds/mem/HugeArrays.java @@ -51,7 +51,7 @@ public static int exclusiveIndexOfPage(long index) { } public static long indexFromPageIndexAndIndexInPage(int pageIndex, int indexInPage) { - return (pageIndex << PAGE_SHIFT) | indexInPage; + return ((long) pageIndex << PAGE_SHIFT) | indexInPage; } public static int numberOfPages(long capacity) { diff --git a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java index 2f189684fa4..74b7162a8fe 100644 --- a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java +++ b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java @@ -27,8 +27,8 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.config.RandomGraphGeneratorConfig.AllowSelfLoops; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.huge.HugeGraph; @@ -143,7 +143,7 @@ public HugeGraph generate() { var relationships = relationshipsBuilder.build(); - var graphSchema = GraphSchema.of( + var graphSchema = MutableGraphSchema.of( nodePropertiesAndSchema.nodeSchema(), relationships.relationshipSchema(relationshipType), Map.of() @@ -226,14 +226,14 @@ private void addDagRelationship(RelationshipsBuilder relationshipsImporter, @ValueClass interface NodePropertiesAndSchema { - NodeSchema nodeSchema(); + MutableNodeSchema nodeSchema(); Map nodeProperties(); } private NodePropertiesAndSchema generateNodeProperties(IdMap idMap) { if (this.nodePropertyProducers.isEmpty()) { - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); idMap.availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); return ImmutableNodePropertiesAndSchema.builder() .nodeSchema(nodeSchema) @@ -284,7 +284,7 @@ private NodePropertiesAndSchema generateNodeProperties(IdMap idMap) { )); // Create a corresponding node schema - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); generatedProperties.forEach((propertyKey, property) -> propertyNameToLabels .get(propertyKey) .forEach(nodeLabel -> { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java index ef302014153..a97f445a013 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java @@ -46,9 +46,10 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.huge.CSRCompositeRelationshipIterator; import org.neo4j.gds.core.huge.HugeGraphBuilder; import org.neo4j.gds.core.huge.NodeFilteredGraph; @@ -90,7 +91,7 @@ public class CSRGraphStore implements GraphStore { private final Set createdGraphs; - private GraphSchema schema; + private MutableGraphSchema schema; private GraphPropertyStore graphProperties; @@ -102,7 +103,7 @@ public class CSRGraphStore implements GraphStore { public static CSRGraphStore of( DatabaseId databaseId, Capabilities capabilities, - GraphSchema schema, + MutableGraphSchema schema, Nodes nodes, RelationshipImportResult relationshipImportResult, Optional graphProperties, @@ -123,7 +124,7 @@ public static CSRGraphStore of( private CSRGraphStore( DatabaseId databaseId, Capabilities capabilities, - GraphSchema schema, + MutableGraphSchema schema, IdMap nodes, NodePropertyStore nodeProperties, Map relationships, @@ -213,7 +214,7 @@ public void addGraphProperty(String propertyKey, GraphPropertyValues propertyVal var newGraphPropertySchema = new HashMap<>(schema().graphProperties()); newGraphPropertySchema.put(propertyKey, PropertySchema.of(propertyKey, propertyValues.valueType())); - this.schema = GraphSchema.of(schema().nodeSchema(), schema().relationshipSchema(), newGraphPropertySchema); + this.schema = MutableGraphSchema.of(schema.nodeSchema(), schema.relationshipSchema(), newGraphPropertySchema); }); } @@ -229,7 +230,7 @@ public void removeGraphProperty(String propertyKey) { var newGraphPropertySchema = new HashMap<>(schema().graphProperties()); newGraphPropertySchema.remove(propertyKey); - this.schema = GraphSchema.of(schema().nodeSchema(), schema().relationshipSchema(), newGraphPropertySchema); + this.schema = MutableGraphSchema.of(schema.nodeSchema(), schema.relationshipSchema(), newGraphPropertySchema); }); } @@ -414,7 +415,7 @@ public void addRelationshipType( ) { updateGraphStore(graphStore -> { graphStore.relationships.computeIfAbsent(relationshipType, __ -> { - var relationshipSchemaEntry = schema() + var relationshipSchemaEntry = schema .relationshipSchema() .getOrCreateRelationshipType(relationshipType, relationships.direction()); relationships.updateRelationshipSchemaEntry(relationshipSchemaEntry); @@ -452,7 +453,7 @@ public DeletionResult deleteRelationships(RelationshipType relationshipType) { property.values().elementCount() )); }); - schema().relationshipSchema().remove(relationshipType); + schema.relationshipSchema().remove(relationshipType); }, () -> builder.deletedRelationships(0)); })); } @@ -598,7 +599,7 @@ private CSRGraph createGraph( ) { var filteredNodes = getFilteredIdMap(nodeLabels); Map filteredNodeProperties = filterNodeProperties(nodeLabels); - var nodeSchema = schema().nodeSchema().filter(new HashSet<>(nodeLabels)); + var nodeSchema = schema.nodeSchema().filter(new HashSet<>(nodeLabels)); return createGraphFromRelationshipType(filteredNodes, filteredNodeProperties, nodeSchema, @@ -614,7 +615,7 @@ private CSRGraph createGraph( ) { var filteredNodes = getFilteredIdMap(filteredLabels); Map filteredNodeProperties = filterNodeProperties(filteredLabels); - var nodeSchema = schema().nodeSchema().filter(new HashSet<>(filteredLabels)); + var nodeSchema = schema.nodeSchema().filter(new HashSet<>(filteredLabels)); List filteredGraphs = relationships .keySet() @@ -636,9 +637,9 @@ private CSRGraph createGraph( private CSRGraph createNodeOnlyGraph(Collection nodeLabels) { var filteredNodes = getFilteredIdMap(nodeLabels); var filteredNodeProperties = filterNodeProperties(nodeLabels); - var nodeSchema = schema().nodeSchema().filter(new HashSet<>(nodeLabels)); + var nodeSchema = schema.nodeSchema().filter(new HashSet<>(nodeLabels)); - var graphSchema = GraphSchema.of(nodeSchema, RelationshipSchema.empty(), schema.graphProperties()); + var graphSchema = MutableGraphSchema.of(nodeSchema, MutableRelationshipSchema.empty(), schema.graphProperties()); var initialGraph = new HugeGraphBuilder() .nodes(nodes) @@ -663,12 +664,12 @@ private Optional getFilteredIdMap(Collection private CSRGraph createGraphFromRelationshipType( Optional filteredNodes, Map filteredNodeProperties, - NodeSchema nodeSchema, + MutableNodeSchema nodeSchema, RelationshipType relationshipType, Optional maybeRelationshipProperty ) { - var graphSchema = GraphSchema.of(nodeSchema, - schema().relationshipSchema().filter(Set.of(relationshipType)), + var graphSchema = MutableGraphSchema.of(nodeSchema, + schema.relationshipSchema().filter(Set.of(relationshipType)), schema.graphProperties() ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index c45d86ce69d..e4f348110ac 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -33,10 +33,10 @@ import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.values.storable.NumberType; @@ -57,7 +57,7 @@ public static CSRGraphStore createFromGraph( var relationshipType = RelationshipType.of(relationshipTypeString); Direction direction = graph.schema().isUndirected() ? Direction.UNDIRECTED : Direction.DIRECTED; - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); var entry = relationshipSchema.getOrCreateRelationshipType(relationshipType, direction); relationshipPropertyKey.ifPresent(property -> { @@ -94,7 +94,7 @@ public static CSRGraphStore createFromGraph( .build() ).build(); - var schema = GraphSchema.of(NodeSchema.from(graph.schema().nodeSchema()), relationshipSchema, Map.of()); + var schema = MutableGraphSchema.of(MutableNodeSchema.from(graph.schema().nodeSchema()), relationshipSchema, Map.of()); return new GraphStoreBuilder() .databaseId(databaseId) @@ -178,12 +178,12 @@ private static Optional constructRelationshipProperti } // TODO: remove this method - public static GraphSchema computeGraphSchema( + public static MutableGraphSchema computeGraphSchema( Nodes nodes, RelationshipImportResult relationshipImportResult ) { var nodeSchema = nodes.schema(); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipImportResult.importResults().forEach(((relationshipType, singleTypeRelationshipImportResult) -> { relationshipSchema.getOrCreateRelationshipType( @@ -200,7 +200,7 @@ public static GraphSchema computeGraphSchema( .addProperty(propertyKey, propertyValues.propertySchema()))); })); - return GraphSchema.of( + return MutableGraphSchema.of( nodeSchema, relationshipSchema, Map.of() diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index 651cd470436..4587dee0ccd 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -23,7 +23,7 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodeLabelToken; import org.neo4j.gds.core.loading.construction.NodesBuilder; @@ -109,7 +109,7 @@ public interface HighLimitIdMapAndProperties { PartialIdMap intermediateIdMap(); - NodeSchema schema(); + MutableNodeSchema schema(); NodePropertyStore propertyStore(); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java index 8aee754ca22..c35d9929221 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/Nodes.java @@ -31,14 +31,13 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.ImmutablePropertySchema; import org.neo4j.gds.api.schema.MutableNodeSchema; -import org.neo4j.gds.api.schema.NodeSchema; import java.util.Map; @ValueClass public interface Nodes { - NodeSchema schema(); + MutableNodeSchema schema(); IdMap idMap(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java index 4c9a0ad6aec..8991a5f5e37 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java @@ -27,9 +27,9 @@ import org.neo4j.gds.api.RelationshipPropertyStore; import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; -import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import java.util.Optional; @@ -53,13 +53,13 @@ public interface SingleTypeRelationships { Optional inverseProperties(); - default RelationshipSchema relationshipSchema(RelationshipType relationshipType) { - var schema = RelationshipSchema.empty(); + default MutableRelationshipSchema relationshipSchema(RelationshipType relationshipType) { + var schema = MutableRelationshipSchema.empty(); this.updateRelationshipSchemaEntry(schema.getOrCreateRelationshipType(relationshipType, direction())); return schema; } - default void updateRelationshipSchemaEntry(RelationshipSchemaEntry schemaEntry) { + default void updateRelationshipSchemaEntry(MutableRelationshipSchemaEntry schemaEntry) { properties().ifPresent(relationshipPropertyStore -> relationshipPropertyStore .relationshipProperties() .forEach((propertyKey, relationshipProperty) -> { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index fe0e35d4f26..e52909d7275 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -34,6 +34,8 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.NodeSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; @@ -298,7 +300,7 @@ static RelationshipsBuilder relationshipsBuilder( * will be used. */ public static HugeGraph create(IdMap idMap, SingleTypeRelationships relationships) { - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); idMap.availableNodeLabels().forEach(nodeSchema::getOrCreateLabel); relationships.properties().ifPresent(relationshipPropertyStore -> { @@ -308,7 +310,7 @@ public static HugeGraph create(IdMap idMap, SingleTypeRelationships relationship var relationshipSchema = relationships.relationshipSchema(RelationshipType.of("REL")); return create( - GraphSchema.of(nodeSchema, relationshipSchema, Map.of()), + MutableGraphSchema.of(nodeSchema, relationshipSchema, Map.of()), idMap, Map.of(), relationships diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java index a36eb4e64c6..9dad02bc635 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/NodesBuilder.java @@ -25,7 +25,7 @@ import org.neo4j.gds.api.properties.nodes.ImmutableNodeProperty; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.compat.LongPropertyReference; import org.neo4j.gds.compat.PropertyReference; @@ -214,7 +214,7 @@ private List closeThreadLocalBuilders() { return labelTokenToPropertyKeys; } - private NodeSchema buildNodeSchema( + private MutableNodeSchema buildNodeSchema( IdMap idMap, Collection localLabelTokenToPropertyKeys, Map nodeProperties @@ -241,7 +241,7 @@ private NodeSchema buildNodeSchema( // key mapping to construct the final node schema. return nodeLabels.stream() .reduce( - NodeSchema.empty(), + MutableNodeSchema.empty(), (unionSchema, nodeLabel) -> unionSchema.addLabel( nodeLabel, globalLabelTokenToPropertyKeys.propertySchemas(nodeLabel, propertyKeysToSchema) diff --git a/core/src/test/java/org/neo4j/gds/beta/generator/RandomGraphGeneratorTest.java b/core/src/test/java/org/neo4j/gds/beta/generator/RandomGraphGeneratorTest.java index aedf8207eb3..5914909d566 100644 --- a/core/src/test/java/org/neo4j/gds/beta/generator/RandomGraphGeneratorTest.java +++ b/core/src/test/java/org/neo4j/gds/beta/generator/RandomGraphGeneratorTest.java @@ -31,8 +31,8 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.config.RandomGraphGeneratorConfig; import org.neo4j.gds.config.RandomGraphGeneratorConfig.AllowSelfLoops; import org.neo4j.gds.core.Aggregation; @@ -465,8 +465,8 @@ void shouldProduceCorrectRelationshipCountWithAggregation() { void shouldProduceCorrectSchema( String name, RandomGraphGenerator generator, - NodeSchema expectedNodeSchema, - RelationshipSchema expectedRelationshipSchema + MutableNodeSchema expectedNodeSchema, + MutableRelationshipSchema expectedRelationshipSchema ) { var graph = generator.generate(); @@ -518,8 +518,8 @@ static Stream expectedSchemas() { .averageDegree(1) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.ALL_NODES), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.ALL_NODES), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) ), Arguments.of("node label", RandomGraphGenerator .builder() @@ -528,8 +528,8 @@ static Stream expectedSchemas() { .nodeLabelProducer(nodeId -> NodeLabelTokens.ofNodeLabels(NodeLabel.of("A"))) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) ), Arguments.of("relationship type", RandomGraphGenerator .builder() @@ -538,8 +538,8 @@ static Stream expectedSchemas() { .relationshipType(RelationshipType.of("FOOBAR")) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.ALL_NODES), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("FOOBAR"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.ALL_NODES), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("FOOBAR"), Direction.DIRECTED) ), Arguments.of("node label and relationship type", RandomGraphGenerator .builder() @@ -549,8 +549,8 @@ static Stream expectedSchemas() { .relationshipType(RelationshipType.of("FOOBAR")) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("FOOBAR"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("FOOBAR"), Direction.DIRECTED) ), Arguments.of("node label and node property", RandomGraphGenerator .builder() @@ -560,8 +560,8 @@ static Stream expectedSchemas() { .nodePropertyProducer(PropertyProducer.randomLong("nodeProp", 0, 42)) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")).addProperty(NodeLabel.of("A"), "nodeProp", ValueType.LONG), - RelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")).addProperty(NodeLabel.of("A"), "nodeProp", ValueType.LONG), + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("REL"), Direction.DIRECTED) ), Arguments.of("relationship type and node property", RandomGraphGenerator .builder() @@ -571,8 +571,8 @@ static Stream expectedSchemas() { .relationshipPropertyProducer(PropertyProducer.randomDouble("relProperty", 0, 42)) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.ALL_NODES), - RelationshipSchema.empty().addProperty(RelationshipType.of("FOOBAR"), + MutableNodeSchema.empty().addLabel(NodeLabel.ALL_NODES), + MutableRelationshipSchema.empty().addProperty(RelationshipType.of("FOOBAR"), Direction.DIRECTED, "relProperty", ValueType.DOUBLE, @@ -589,8 +589,8 @@ static Stream expectedSchemas() { .relationshipPropertyProducer(PropertyProducer.randomDouble("relProp", 0, 42)) .relationshipDistribution(RelationshipDistribution.RANDOM) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")).addProperty(NodeLabel.of("A"), "nodeProp", ValueType.LONG), - RelationshipSchema.empty().addProperty(RelationshipType.of("FOOBAR"), + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")).addProperty(NodeLabel.of("A"), "nodeProp", ValueType.LONG), + MutableRelationshipSchema.empty().addProperty(RelationshipType.of("FOOBAR"), Direction.DIRECTED, "relProp", ValueType.DOUBLE, @@ -607,8 +607,8 @@ static Stream expectedSchemas() { .relationshipDistribution(RelationshipDistribution.RANDOM) .direction(UNDIRECTED) .build(), - NodeSchema.empty().addLabel(NodeLabel.of("A")), - RelationshipSchema + MutableNodeSchema.empty().addLabel(NodeLabel.of("A")), + MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("FOOBAR"), UNDIRECTED, diff --git a/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java b/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java index 2ddf4329989..28847e38213 100644 --- a/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java @@ -23,8 +23,8 @@ import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.loading.GraphStoreBuilder; @@ -47,13 +47,13 @@ void validateGraphStoreCapabilities(boolean isBackedByDatabase) { var nodes = ImmutableNodes.builder() .idMap(new DirectIdMap(0)) - .schema(NodeSchema.empty()) + .schema(MutableNodeSchema.empty()) .build(); var testGraphStore = new GraphStoreBuilder() .databaseId(DatabaseId.from("neo4j")) .capabilities(ImmutableStaticCapabilities.of(isBackedByDatabase)) - .schema(GraphSchema.empty()) + .schema(MutableGraphSchema.empty()) .nodes(nodes) .relationshipImportResult(RelationshipImportResult.of(Map.of())) .concurrency(1) diff --git a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java index ef844f9c34b..2de63acfcbc 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/construction/NodeLabelTokenToPropertyKeysTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import java.util.List; @@ -65,7 +65,7 @@ void testPropertySchemasLazy() { @Test void testPropertySchemasFixed() { - var nodeSchema = NodeSchema.empty() + var nodeSchema = MutableNodeSchema.empty() .addLabel(NodeLabel.of("A"), Map.of( "foo", PropertySchema.of("foo", ValueType.LONG), "bar", PropertySchema.of("bar", ValueType.DOUBLE), @@ -100,7 +100,7 @@ void testPropertySchemasFixed() { @Test void shouldFailForMissingProperties() { - var nodeSchema = NodeSchema.empty() + var nodeSchema = MutableNodeSchema.empty() .addLabel(NodeLabel.of("A"), Map.of( "foo", PropertySchema.of("foo", ValueType.LONG), "baz", PropertySchema.of("baz", ValueType.LONG) @@ -121,7 +121,7 @@ void shouldFailForMissingProperties() { @Test void shouldFailForIncompatibleTypes() { - var nodeSchema = NodeSchema.empty() + var nodeSchema = MutableNodeSchema.empty() .addLabel(NodeLabel.of("A"), Map.of( "foo", PropertySchema.of("foo", ValueType.LONG), "baz", PropertySchema.of("baz", ValueType.LONG) @@ -195,9 +195,19 @@ void testNodeLabelsLazily() { @Test void testNodeLabelsFixed() { - var mapping = NodeLabelTokenToPropertyKeys.fixed(NodeSchema.empty().addLabel(NodeLabel.ALL_NODES).addLabel(NodeLabel.of("A")).addLabel(NodeLabel.of("B")).addLabel(NodeLabel.of("C"))); + var mapping = NodeLabelTokenToPropertyKeys.fixed(MutableNodeSchema + .empty() + .addLabel(NodeLabel.ALL_NODES) + .addLabel(NodeLabel.of("A")) + .addLabel(NodeLabel.of("B")) + .addLabel(NodeLabel.of("C"))); - assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder(NodeLabel.ALL_NODES, NodeLabel.of("A"), NodeLabel.of("B"), NodeLabel.of("C")); + assertThat(mapping.nodeLabels()).containsExactlyInAnyOrder( + NodeLabel.ALL_NODES, + NodeLabel.of("A"), + NodeLabel.of("B"), + NodeLabel.of("C") + ); } @Test diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index 2e154e30b58..fdf82225a33 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -25,9 +25,10 @@ import org.neo4j.gds.annotation.CustomProcedure; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.DefaultValue; -import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; -import org.neo4j.gds.api.schema.RelationshipSchemaEntry; +import org.neo4j.gds.api.schema.ImmutableMutableGraphSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.gds.compat.CompatUserAggregator; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.ConfigKeyValidation; @@ -314,7 +315,7 @@ private static final class GraphImporter { private final boolean canWriteToDatabase; private final ExtractNodeId extractNodeId; private final Map relImporters; - private final ImmutableGraphSchema.Builder graphSchemaBuilder; + private final ImmutableMutableGraphSchema.Builder graphSchemaBuilder; private GraphImporter( String graphName, @@ -327,7 +328,7 @@ private GraphImporter( this.idMapBuilder = idMapBuilder; this.canWriteToDatabase = canWriteToDatabase; this.relImporters = new ConcurrentHashMap<>(); - this.graphSchemaBuilder = ImmutableGraphSchema.builder(); + this.graphSchemaBuilder = MutableGraphSchema.builder(); this.extractNodeId = new ExtractNodeId(); } @@ -531,7 +532,7 @@ private void buildRelationshipsWithProperties( AdjacencyCompressor.ValueMapper valueMapper ) { var relationshipImportResultBuilder = RelationshipImportResult.builder(); - var relationshipSchemas = new HashMap(); + var relationshipSchemas = new HashMap(); this.relImporters.forEach((relationshipType, relImporter) -> { var relationships = relImporter.build( @@ -543,7 +544,7 @@ private void buildRelationshipsWithProperties( relationshipImportResultBuilder.putImportResult(relationshipType, relationships); }); - var relationshipSchema = new RelationshipSchema(relationshipSchemas); + var relationshipSchema = new MutableRelationshipSchema(relationshipSchemas); graphStoreBuilder.relationshipImportResult(relationshipImportResultBuilder.build()); this.graphSchemaBuilder.relationshipSchema(relationshipSchema); diff --git a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/GraphSchemaTest.java b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/GraphSchemaTest.java index 1e7d94ba085..5c4159aec51 100644 --- a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/GraphSchemaTest.java +++ b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/GraphSchemaTest.java @@ -36,21 +36,21 @@ void testUnionOfGraphPropertiesForDifferentProperties() { var prop1Schema = PropertySchema.of("prop1", ValueType.LONG, DefaultValue.of(42L), PropertyState.TRANSIENT); var prop2Schema = PropertySchema.of("prop2", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.TRANSIENT); - var graphSchema1 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema1 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema) ); - var graphSchema2 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema2 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop2", prop2Schema) ); - var graphSchemaUnion = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchemaUnion = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of( "prop1", prop1Schema, "prop2", prop2Schema @@ -64,15 +64,15 @@ void testUnionOfGraphPropertiesForDifferentProperties() { void testUnionOfGraphPropertiesForSameProperties() { var prop1Schema = PropertySchema.of("prop1", ValueType.LONG, DefaultValue.of(42L), PropertyState.TRANSIENT); - var graphSchema1 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema1 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema) ); - var graphSchema2 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema2 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema) ); @@ -84,15 +84,15 @@ void testUnionOfGraphPropertiesForSamePropertiesWithDifferentTypeThrows() { var prop1Schema1 = PropertySchema.of("prop1", ValueType.LONG, DefaultValue.of(42L), PropertyState.TRANSIENT); var prop1Schema2 = PropertySchema.of("prop1", ValueType.DOUBLE, DefaultValue.forDouble(), PropertyState.TRANSIENT); - var graphSchema1 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema1 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema1) ); - var graphSchema2 = GraphSchema.of( - NodeSchema.empty(), - RelationshipSchema.empty(), + var graphSchema2 = MutableGraphSchema.of( + MutableNodeSchema.empty(), + MutableRelationshipSchema.empty(), Map.of("prop1", prop1Schema2) ); diff --git a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/NodeSchemaTest.java b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/NodeSchemaTest.java index 8776e4bdb6e..8b23148b83e 100644 --- a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/NodeSchemaTest.java +++ b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/NodeSchemaTest.java @@ -42,7 +42,7 @@ class NodeSchemaTest { @Test void handlesOutsideOfSchemaRequests() { - NodeSchema empty = NodeSchema.empty(); + MutableNodeSchema empty = MutableNodeSchema.empty(); assertFalse(empty.hasProperty(NodeLabel.of("NotInSchema"), "notInSchemaEither")); } @@ -52,7 +52,7 @@ void testDefaultValues() { DefaultValue defaultValue = DefaultValue.of(42.0D); String propertyName = "baz"; - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema .getOrCreateLabel(label) .addProperty( @@ -70,13 +70,13 @@ void testFiltering() { var label1 = NodeLabel.of("Foo"); var label2 = NodeLabel.of("Bar"); - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE).addProperty("baz", ValueType.DOUBLE); nodeSchema.getOrCreateLabel(label2).addProperty("baz", ValueType.DOUBLE); assertEquals(nodeSchema, nodeSchema.filter(Set.of(label1, label2))); - var expected = NodeSchema.empty(); + var expected = MutableNodeSchema.empty(); expected.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE).addProperty("baz", ValueType.DOUBLE); assertEquals(expected, nodeSchema.filter(Set.of(label1))); @@ -87,13 +87,13 @@ void testUnion() { var label1 = NodeLabel.of("Foo"); var label2 = NodeLabel.of("Bar"); - var nodeSchema1 = NodeSchema.empty(); + var nodeSchema1 = MutableNodeSchema.empty(); nodeSchema1.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE); - var nodeSchema2 = NodeSchema.empty(); + var nodeSchema2 = MutableNodeSchema.empty(); nodeSchema2.getOrCreateLabel(label2).addProperty("bar", ValueType.DOUBLE); - var expected = NodeSchema.empty(); + var expected = MutableNodeSchema.empty(); expected.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE); expected.getOrCreateLabel(label2).addProperty("bar", ValueType.DOUBLE); @@ -105,15 +105,15 @@ void testUnionSchema() { var label1 = NodeLabel.of("Foo"); var label2 = NodeLabel.of("Bar"); - var nodeSchema1 = NodeSchema.empty(); + var nodeSchema1 = MutableNodeSchema.empty(); nodeSchema1.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE); nodeSchema1.getOrCreateLabel(label2); - var nodeSchema2 = NodeSchema.empty(); + var nodeSchema2 = MutableNodeSchema.empty(); nodeSchema2.getOrCreateLabel(label1).addProperty("baz", ValueType.DOUBLE); nodeSchema2.getOrCreateLabel(label2).addProperty("baz", ValueType.DOUBLE); - var expected = NodeSchema.empty(); + var expected = MutableNodeSchema.empty(); expected.getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE).addProperty("baz", ValueType.DOUBLE); expected.getOrCreateLabel(label2).addProperty("baz", ValueType.DOUBLE); @@ -124,10 +124,10 @@ void testUnionSchema() { void testUnionOfIncompatibleProperties() { var label1 = NodeLabel.of("Foo"); - var nodeSchema1 = NodeSchema.empty() + var nodeSchema1 = MutableNodeSchema.empty() .getOrCreateLabel(label1).addProperty("bar", ValueType.DOUBLE); - var nodeSchema2 = NodeSchema.empty() + var nodeSchema2 = MutableNodeSchema.empty() .getOrCreateLabel(label1).addProperty("bar", ValueType.LONG); var ex = assertThrows(IllegalArgumentException.class, () -> nodeSchema1.union(nodeSchema2)); @@ -141,7 +141,7 @@ void testUnionProperties() { var label1 = NodeLabel.of("Foo"); var label2 = NodeLabel.of("Bar"); - var nodeSchema = NodeSchema.empty() + var nodeSchema = MutableNodeSchema.empty() .addProperty(label1, "foo", ValueType.DOUBLE) .addProperty(label1, "baz", ValueType.LONG) .addProperty(label2, "bar", ValueType.LONG_ARRAY) @@ -164,7 +164,7 @@ void testUnionProperties() { @Test void shouldCreateDeepCopiesWhenFiltering() { var nodeLabel = NodeLabel.of("A"); - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema.getOrCreateLabel(nodeLabel).addProperty("prop", ValueType.LONG); var filteredNodeSchema = nodeSchema.filter(Set.of(nodeLabel)); filteredNodeSchema @@ -176,17 +176,17 @@ void shouldCreateDeepCopiesWhenFiltering() { } static Stream schemaAndHasProperties() { - NodeSchema.empty(); + MutableNodeSchema.empty(); return Stream.of( - Arguments.of(NodeSchema.empty().addLabel(NodeLabel.of("A")), false), - Arguments.of(NodeSchema.empty().addProperty(NodeLabel.of("A"), "foo", ValueType.LONG), true) + Arguments.of(MutableNodeSchema.empty().addLabel(NodeLabel.of("A")), false), + Arguments.of(MutableNodeSchema.empty().addProperty(NodeLabel.of("A"), "foo", ValueType.LONG), true) ); } @ParameterizedTest @MethodSource("schemaAndHasProperties") - void shouldKnowIfPropertiesArePresent(NodeSchema nodeSchema, boolean hasProperties) { + void shouldKnowIfPropertiesArePresent(MutableNodeSchema nodeSchema, boolean hasProperties) { assertThat(nodeSchema.hasProperties()).isEqualTo(hasProperties); } @@ -194,7 +194,7 @@ void shouldKnowIfPropertiesArePresent(NodeSchema nodeSchema, boolean hasProperti void shouldAddNodeLabelWithNodeProperties() { var label = NodeLabel.of("Foo"); - var nodeSchema = NodeSchema.empty() + var nodeSchema = MutableNodeSchema.empty() .addProperty(label, "foo", ValueType.DOUBLE) .addProperty(label, "baz", PropertySchema.of("baz", ValueType.LONG, DefaultValue.forLong(), PropertyState.TRANSIENT)) .addProperty(label, "bar", ValueType.LONG_ARRAY); diff --git a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java index a5f5065a73c..8b62f646777 100644 --- a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java +++ b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java @@ -41,7 +41,7 @@ class RelationshipSchemaTest { @Test void inAndOutUndirected() { - assertThat(RelationshipSchema.empty().isUndirected()).isTrue(); + assertThat(MutableRelationshipSchema.empty().isUndirected()).isTrue(); RelationshipType type = RelationshipType.of("TYPE"); var propertySchema = Map.of( @@ -49,23 +49,23 @@ void inAndOutUndirected() { Map.of("prop", RelationshipPropertySchema.of("prop", ValueType.DOUBLE)) ); - var undirectedSchema = RelationshipSchema.empty(); + var undirectedSchema = MutableRelationshipSchema.empty(); undirectedSchema.getOrCreateRelationshipType(type, Direction.UNDIRECTED); assertThat(undirectedSchema.isUndirected()).isTrue(); - var directedSchema = RelationshipSchema.empty(); + var directedSchema = MutableRelationshipSchema.empty(); directedSchema.getOrCreateRelationshipType(type, Direction.DIRECTED); assertThat(directedSchema.isUndirected()).isFalse(); } @Test void emptyIsUndirected() { - assertThat(RelationshipSchema.empty().isUndirected()).isTrue(); + assertThat(MutableRelationshipSchema.empty().isUndirected()).isTrue(); } @Test void handlesOutsideOfSchemaRequests() { - var empty = RelationshipSchema.empty(); + var empty = MutableRelationshipSchema.empty(); assertThat(empty.hasProperty(RelationshipType.of("NotInSchema"), "notInSchemaEither")).isFalse(); } @@ -77,7 +77,7 @@ void testDefaultValuesAndAggregation() { Aggregation aggregation = Aggregation.COUNT; PropertyState propertyState = PropertyState.PERSISTENT; String propertyName = "baz"; - var relationshipSchema = RelationshipSchema.empty() + var relationshipSchema = MutableRelationshipSchema.empty() .addProperty( relType, Direction.DIRECTED, @@ -115,14 +115,14 @@ void testFiltering(Direction orientation, boolean isUndirected) { var label1 = RelationshipType.of("Foo"); var label2 = RelationshipType.of("Bar"); - var relationshipSchema = RelationshipSchema.empty() + var relationshipSchema = MutableRelationshipSchema.empty() .addProperty(label1, orientation, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(label1, orientation, "baz", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(label2, orientation, "baz", ValueType.DOUBLE, PropertyState.PERSISTENT); assertThat(relationshipSchema.filter(Set.of(label1, label2))).isEqualTo(relationshipSchema); - var expected = RelationshipSchema.empty() + var expected = MutableRelationshipSchema.empty() .addProperty(label1, orientation, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(label1, orientation, "baz", ValueType.DOUBLE, PropertyState.PERSISTENT); @@ -135,9 +135,9 @@ void testFilteringMixed() { var directedType = RelationshipType.of("D"); var undirectedType = RelationshipType.of("U"); - var directed = RelationshipSchema.empty() + var directed = MutableRelationshipSchema.empty() .addProperty(directedType, Direction.DIRECTED, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT); - var undirected = RelationshipSchema.empty() + var undirected = MutableRelationshipSchema.empty() .addProperty(undirectedType, Direction.UNDIRECTED, "flob", ValueType.DOUBLE, PropertyState.PERSISTENT); var mixed = directed.union(undirected); @@ -164,13 +164,13 @@ void testUnion(Direction direction1, Direction direction2, Boolean isUndirectedE var type1 = RelationshipType.of("Foo"); var type2 = RelationshipType.of("Bar"); - var relationshipSchema1 = RelationshipSchema.empty() + var relationshipSchema1 = MutableRelationshipSchema.empty() .addProperty(type1, direction1, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT); - var relationshipSchema2 = RelationshipSchema.empty() + var relationshipSchema2 = MutableRelationshipSchema.empty() .addProperty(type2, direction2, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT); - var expected = RelationshipSchema.empty() + var expected = MutableRelationshipSchema.empty() .addProperty(type1, direction1, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(type2, direction2, "bar", ValueType.DOUBLE, PropertyState.PERSISTENT); @@ -184,11 +184,11 @@ void testUnion(Direction direction1, Direction direction2, Boolean isUndirectedE @Test void unionOnSameTypesFailsOnDirectionMismatch() { - var schema1 = RelationshipSchema.empty() + var schema1 = MutableRelationshipSchema.empty() .addRelationshipType(RelationshipType.of("X"), Direction.DIRECTED) .addRelationshipType(RelationshipType.of("Y"), Direction.UNDIRECTED); - var schema2 = RelationshipSchema.empty() + var schema2 = MutableRelationshipSchema.empty() .addRelationshipType(RelationshipType.of("X"), Direction.UNDIRECTED) .addRelationshipType(RelationshipType.of("Y"), Direction.UNDIRECTED); @@ -199,7 +199,7 @@ void unionOnSameTypesFailsOnDirectionMismatch() { @Test void unionOnSameTypeSamePropertyDifferentValueTypeFails() { - var schema1 = RelationshipSchema + var schema1 = MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("X"), Direction.DIRECTED, "x", ValueType.DOUBLE, PropertyState.PERSISTENT) .addProperty(RelationshipType.of("Y"), @@ -209,7 +209,7 @@ void unionOnSameTypeSamePropertyDifferentValueTypeFails() { PropertyState.PERSISTENT ); - var schema2 = RelationshipSchema + var schema2 = MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("X"), Direction.DIRECTED, "x", ValueType.LONG, PropertyState.PERSISTENT) .addProperty(RelationshipType.of("Y"), @@ -229,13 +229,15 @@ void unionOnSameTypeSamePropertyDifferentValueTypeFails() { static Stream schemaAndHasProperties() { return Stream.of( - Arguments.of(RelationshipSchema.empty().addRelationshipType(RelationshipType.of("A"), Direction.DIRECTED), + Arguments.of( + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("A"), Direction.DIRECTED), false ), - Arguments.of(RelationshipSchema.empty().addRelationshipType(RelationshipType.of("A"), Direction.UNDIRECTED), + Arguments.of( + MutableRelationshipSchema.empty().addRelationshipType(RelationshipType.of("A"), Direction.UNDIRECTED), false ), - Arguments.of(RelationshipSchema + Arguments.of(MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("A"), Direction.DIRECTED, @@ -243,7 +245,7 @@ static Stream schemaAndHasProperties() { ValueType.LONG, PropertyState.PERSISTENT ), true), - Arguments.of(RelationshipSchema + Arguments.of(MutableRelationshipSchema .empty() .addProperty(RelationshipType.of("A"), Direction.UNDIRECTED, @@ -256,13 +258,13 @@ static Stream schemaAndHasProperties() { @ParameterizedTest @MethodSource("schemaAndHasProperties") - void shouldKnowIfPropertiesArePresent(RelationshipSchema relationshipSchema, boolean hasProperties) { + void shouldKnowIfPropertiesArePresent(MutableRelationshipSchema relationshipSchema, boolean hasProperties) { assertThat(relationshipSchema.hasProperties()).isEqualTo(hasProperties); } @Test void testBuildRelTypes() { - var schema = RelationshipSchema.empty(); + var schema = MutableRelationshipSchema.empty(); schema.addRelationshipType(RelationshipType.of("X"), Direction.UNDIRECTED); schema.addRelationshipType(RelationshipType.of("Y"), Direction.DIRECTED); @@ -276,7 +278,7 @@ void testBuildRelTypes() { @Test void testBuildProperties() { - RelationshipSchema relationshipSchema = RelationshipSchema.empty() + MutableRelationshipSchema relationshipSchema = MutableRelationshipSchema.empty() .addProperty(RelationshipType.of("X"), Direction.UNDIRECTED, "x", ValueType.DOUBLE, PropertyState.PERSISTENT ); @@ -316,7 +318,7 @@ void testBuildProperties() { @Test void shouldCreateDeepCopiesWhenFiltering() { var relType = RelationshipType.of("A"); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipSchema.getOrCreateRelationshipType(relType, Direction.DIRECTED).addProperty("prop", ValueType.LONG, PropertyState.PERSISTENT ); diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java index 1b9ae3f02a1..5122e3a5e50 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileInput.java @@ -19,9 +19,9 @@ */ package org.neo4j.gds.core.io.file; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.compat.CompatInput; import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.internal.batchimport.InputIterable; @@ -32,8 +32,8 @@ public interface FileInput extends CompatInput { InputIterable graphProperties(); String userName(); GraphInfo graphInfo(); - NodeSchema nodeSchema(); - RelationshipSchema relationshipSchema(); + MutableNodeSchema nodeSchema(); + MutableRelationshipSchema relationshipSchema(); Map graphPropertySchema(); Capabilities capabilities(); } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 00f9af3a861..9ca4493139d 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -26,8 +26,8 @@ import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.RelationshipPropertyStore; import org.neo4j.gds.api.Topology; -import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.ImmutableMutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.concurrency.ParallelUtil; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.io.GraphStoreGraphPropertyVisitor; @@ -64,7 +64,7 @@ public abstract class FileToGraphStoreImporter { private final Path importPath; private final int concurrency; - private final ImmutableGraphSchema.Builder graphSchemaBuilder; + private final ImmutableMutableGraphSchema.Builder graphSchemaBuilder; private final GraphStoreBuilder graphStoreBuilder; private final Log log; private final TaskRegistryFactory taskRegistryFactory; @@ -82,7 +82,7 @@ protected FileToGraphStoreImporter( this.graphPropertyVisitorBuilder = new GraphStoreGraphPropertyVisitor.Builder(); this.concurrency = concurrency; this.importPath = importPath; - this.graphSchemaBuilder = ImmutableGraphSchema.builder(); + this.graphSchemaBuilder = ImmutableMutableGraphSchema.builder(); this.graphStoreBuilder = new GraphStoreBuilder() .concurrency(concurrency) // TODO: we need to export and import this flag: https://trello.com/c/2cEMPZ9L @@ -143,7 +143,7 @@ private void importGraphStore(FileInput fileInput) { private Nodes importNodes(FileInput fileInput) { progressTracker.beginSubTask(); - NodeSchema nodeSchema = fileInput.nodeSchema(); + MutableNodeSchema nodeSchema = fileInput.nodeSchema(); graphSchemaBuilder.nodeSchema(nodeSchema); NodesBuilder nodesBuilder = GraphFactory.initNodesBuilder(nodeSchema) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitor.java index bf125ad3290..e8b12ed8c29 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitor.java @@ -19,15 +19,15 @@ */ package org.neo4j.gds.core.io.schema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.PropertySchema; public class NodeSchemaBuilderVisitor extends NodeSchemaVisitor { - private final NodeSchema nodeSchema; + private final MutableNodeSchema nodeSchema; public NodeSchemaBuilderVisitor() { - nodeSchema = NodeSchema.empty(); + nodeSchema = MutableNodeSchema.empty(); } @Override @@ -42,7 +42,7 @@ protected void export() { } } - public NodeSchema schema() { + public MutableNodeSchema schema() { return nodeSchema; } } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/schema/RelationshipSchemaBuilderVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/schema/RelationshipSchemaBuilderVisitor.java index 0abafde4a61..f4fc164a0e3 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/schema/RelationshipSchemaBuilderVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/schema/RelationshipSchemaBuilderVisitor.java @@ -19,15 +19,15 @@ */ package org.neo4j.gds.core.io.schema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; public class RelationshipSchemaBuilderVisitor extends RelationshipSchemaVisitor { - private final RelationshipSchema schema; + private final MutableRelationshipSchema schema; public RelationshipSchemaBuilderVisitor() { - this.schema = RelationshipSchema.empty(); + this.schema = MutableRelationshipSchema.empty(); } @Override @@ -41,7 +41,7 @@ protected void export() { } } - public RelationshipSchema schema() { + public MutableRelationshipSchema schema() { return schema; } } diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreNodeVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreNodeVisitorTest.java index f4bff87c51f..0461ef3fe07 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreNodeVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreNodeVisitorTest.java @@ -25,9 +25,9 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; @@ -56,7 +56,7 @@ class GraphStoreNodeVisitorTest { @Test void shouldAddNodesToNodesBuilder() { - NodeSchema nodeSchema = NodeSchema.from(graphStore.schema().nodeSchema()); + MutableNodeSchema nodeSchema = MutableNodeSchema.from(graphStore.schema().nodeSchema()); NodesBuilder nodesBuilder = GraphFactory.initNodesBuilder(nodeSchema) .concurrency(1) .maxOriginalId(graphStore.nodeCount()) @@ -79,10 +79,10 @@ void shouldAddNodesToNodesBuilder() { var idMap = nodes.idMap(); var nodeProperties = nodes.properties().propertyValues(); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipSchema.getOrCreateRelationshipType(RelationshipType.ALL_RELATIONSHIPS, Direction.UNDIRECTED); - var graphSchema = GraphSchema.of( + var graphSchema = MutableGraphSchema.of( nodeSchema, relationshipSchema, Map.of() diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java index 97abfd9b202..96efedf45b3 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java @@ -26,6 +26,7 @@ import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.RelationshipPropertyStore; import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.io.GraphStoreRelationshipVisitor; import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.ImmutableNodes; @@ -154,7 +155,7 @@ private Graph createGraph( var nodes = ImmutableNodes.builder() .idMap(expectedGraph) - .schema(expectedGraph.schema().nodeSchema()) + .schema(MutableNodeSchema.from(expectedGraph.schema().nodeSchema())) .build(); Map propertyStores = actualRelationships.properties(); diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitorTest.java index 151064d9b78..66de46b4261 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/schema/NodeSchemaBuilderVisitorTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchemaEntry; +import org.neo4j.gds.api.schema.MutableNodeSchemaEntry; import org.neo4j.gds.api.schema.PropertySchema; import java.util.Map; @@ -62,7 +62,7 @@ void shouldBuildNodeSchema() { var labelAEntry = builtSchema.get(NodeLabel.of("A")); assertThat(labelAEntry) .isEqualTo( - new NodeSchemaEntry( + new MutableNodeSchemaEntry( NodeLabel.of("A"), Map.of( "prop1", @@ -79,7 +79,7 @@ void shouldBuildNodeSchema() { var labelBEntry = builtSchema.get(NodeLabel.of("B")); assertThat(labelBEntry) .isEqualTo( - new NodeSchemaEntry( + new MutableNodeSchemaEntry( NodeLabel.of("B"), Map.of( "prop2", diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/NodeFileHeader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/NodeFileHeader.java index edb4f21e027..198dc3636af 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/NodeFileHeader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/NodeFileHeader.java @@ -21,7 +21,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.core.io.file.csv.CsvNodeVisitor; import org.neo4j.gds.utils.StringFormatting; @@ -33,11 +33,11 @@ import java.util.stream.Stream; @ValueClass -public interface NodeFileHeader extends FileHeader { +public interface NodeFileHeader extends FileHeader { String[] nodeLabels(); @Override - default Map schemaForIdentifier(NodeSchema schema) { + default Map schemaForIdentifier(MutableNodeSchema schema) { var labelStream = Arrays.stream(nodeLabels()).map(NodeLabel::of); if (nodeLabels().length == 0) { labelStream = Stream.of(NodeLabel.ALL_NODES); diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/RelationshipFileHeader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/RelationshipFileHeader.java index bb692eee7e8..3442ab2e3c9 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/RelationshipFileHeader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/RelationshipFileHeader.java @@ -21,8 +21,8 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.io.file.csv.CsvRelationshipVisitor; import org.neo4j.gds.utils.StringFormatting; @@ -30,11 +30,11 @@ import java.util.Set; @ValueClass -public interface RelationshipFileHeader extends FileHeader { +public interface RelationshipFileHeader extends FileHeader { String relationshipType(); @Override - default Map schemaForIdentifier(RelationshipSchema schema) { + default Map schemaForIdentifier(MutableRelationshipSchema schema) { return schema.filter(Set.of(RelationshipType.of(relationshipType()))).unionProperties(); } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java index 4b9d67659c1..4a4a776d3f1 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/CsvFileInput.java @@ -25,10 +25,10 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; import org.apache.commons.lang3.tuple.Pair; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.compat.CompatPropertySizeCalculator; import org.neo4j.gds.core.io.GraphStoreInput; import org.neo4j.gds.core.io.file.FileHeader; @@ -79,8 +79,8 @@ final class CsvFileInput implements FileInput { private final Path importPath; private final String userName; private final GraphInfo graphInfo; - private final NodeSchema nodeSchema; - private final RelationshipSchema relationshipSchema; + private final MutableNodeSchema nodeSchema; + private final MutableRelationshipSchema relationshipSchema; private final Map graphPropertySchema; private final Capabilities capabilities; @@ -153,12 +153,12 @@ public GraphInfo graphInfo() { } @Override - public NodeSchema nodeSchema() { + public MutableNodeSchema nodeSchema() { return nodeSchema; } @Override - public RelationshipSchema relationshipSchema() { + public MutableRelationshipSchema relationshipSchema() { return relationshipSchema; } @@ -206,11 +206,11 @@ public void close() { } } - static class NodeImporter extends FileImporter { + static class NodeImporter extends FileImporter { NodeImporter( Map> headerToDataFilesMapping, - NodeSchema nodeSchema + MutableNodeSchema nodeSchema ) { super(headerToDataFilesMapping, nodeSchema); } @@ -221,11 +221,11 @@ public InputChunk newChunk() { } } - static class RelationshipImporter extends FileImporter { + static class RelationshipImporter extends FileImporter { RelationshipImporter( Map> headerToDataFilesMapping, - RelationshipSchema relationshipSchema + MutableRelationshipSchema relationshipSchema ) { super(headerToDataFilesMapping, relationshipSchema); } @@ -301,9 +301,9 @@ public long lastProgress() { } } - static class NodeLineChunk extends LineChunk { + static class NodeLineChunk extends LineChunk { - NodeLineChunk(NodeSchema nodeSchema) { + NodeLineChunk(MutableNodeSchema nodeSchema) { super(nodeSchema); } @@ -318,9 +318,9 @@ void visitLine(String[] lineArray, NodeFileHeader header, InputEntityVisitor vis } } - static class RelationshipLineChunk extends LineChunk { + static class RelationshipLineChunk extends LineChunk { - RelationshipLineChunk(RelationshipSchema relationshipSchema) { + RelationshipLineChunk(MutableRelationshipSchema relationshipSchema) { super(relationshipSchema); } diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java index 3b50879d643..e606f114b9f 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporter.java @@ -22,7 +22,7 @@ import org.jetbrains.annotations.TestOnly; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.io.NeoNodeProperties; import org.neo4j.gds.core.io.file.GraphStoreToFileExporter; import org.neo4j.gds.core.io.file.GraphStoreToFileExporterConfig; @@ -59,7 +59,7 @@ public static GraphStoreToFileExporter create( var nodeSchema = graphStore.schema().nodeSchema(); var relationshipSchema = graphStore.schema().relationshipSchema(); - var neoNodeSchema = NodeSchema.empty(); + var neoNodeSchema = MutableNodeSchema.empty(); // Add additional properties to each label present in the graph store. neoNodeProperties.ifPresent(additionalProps -> additionalProps diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java index c8f891680da..18fbfee3dfc 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoader.java @@ -29,7 +29,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.io.schema.NodeSchemaBuilderVisitor; import java.io.BufferedReader; @@ -51,7 +51,7 @@ public class NodeSchemaLoader { objectReader = csvMapper.readerFor(SchemaLine.class).with(schema); } - NodeSchema load() { + MutableNodeSchema load() { NodeSchemaBuilderVisitor schemaBuilder = new NodeSchemaBuilderVisitor(); try(var reader = new BufferedReader(new FileReader(nodeSchemaPath.toFile(), StandardCharsets.UTF_8))) { diff --git a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java index 07973f7c3c3..d2802af58aa 100644 --- a/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java +++ b/io/csv/src/main/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoader.java @@ -30,7 +30,7 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.io.schema.RelationshipSchemaBuilderVisitor; @@ -53,7 +53,7 @@ public class RelationshipSchemaLoader { objectReader = csvMapper.readerFor(SchemaLine.class).with(schema); } - RelationshipSchema load() { + MutableRelationshipSchema load() { var schemaBuilder = new RelationshipSchemaBuilderVisitor(); try (var reader = new BufferedReader(new FileReader(relationshipSchemaPath.toFile(), StandardCharsets.UTF_8))) { diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitorTest.java index c0c37d6267a..b6b690bf79e 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvNodeVisitorTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; import java.util.Collections; import java.util.List; @@ -34,7 +34,7 @@ class CsvNodeVisitorTest extends CsvVisitorTest { @Test void visitNodesWithoutLabelsAndProperties() { - var nodeVisitor = new CsvNodeVisitor(tempDir, NodeSchema.empty()); + var nodeVisitor = new CsvNodeVisitor(tempDir, MutableNodeSchema.empty()); nodeVisitor.id(0L); nodeVisitor.endOfEntity(); @@ -55,7 +55,7 @@ void visitNodesWithoutLabelsAndProperties() { @Test void visitNodesWithLabels() { - var nodeVisitor = new CsvNodeVisitor(tempDir, NodeSchema.empty()); + var nodeVisitor = new CsvNodeVisitor(tempDir, MutableNodeSchema.empty()); nodeVisitor.id(0L); nodeVisitor.labels(new String[]{"Foo", "Bar"}); @@ -92,7 +92,7 @@ void visitNodesWithLabels() { @Test void visitNodesWithProperties() { - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema.getOrCreateLabel(NodeLabel.ALL_NODES) .addProperty("foo", ValueType.DOUBLE) .addProperty("bar", ValueType.DOUBLE); @@ -131,7 +131,7 @@ void visitNodesWithLabelsAndProperties() { var bLabel = NodeLabel.of("B"); var cLabel = NodeLabel.of("C"); - var nodeSchema = NodeSchema.empty(); + var nodeSchema = MutableNodeSchema.empty(); nodeSchema.getOrCreateLabel(aLabel) .addProperty("foo", ValueType.LONG) .addProperty("bar", ValueType.LONG); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvRelationshipVisitorTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvRelationshipVisitorTest.java index 0544b7deacb..68d243e2054 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvRelationshipVisitorTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvRelationshipVisitorTest.java @@ -24,7 +24,7 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import java.util.Collections; import java.util.List; @@ -37,7 +37,7 @@ class CsvRelationshipVisitorTest extends CsvVisitorTest { @Test void visitRelationshipsWithTypes() { - var relationshipVisitor = new CsvRelationshipVisitor(tempDir, RelationshipSchema.empty()); + var relationshipVisitor = new CsvRelationshipVisitor(tempDir, MutableRelationshipSchema.empty()); relationshipVisitor.startId(0L); relationshipVisitor.endId(1L); @@ -80,7 +80,7 @@ void visitRelationshipsWithTypesAndProperties() { var aType = RelationshipType.of("A"); var bType = RelationshipType.of("B"); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipSchema.getOrCreateRelationshipType(aType, Direction.DIRECTED) .addProperty("foo", ValueType.LONG, PropertyState.PERSISTENT) .addProperty("bar", ValueType.LONG, PropertyState.PERSISTENT); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java index c8365e3b07d..2367508e6c9 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/NodeSchemaLoaderTest.java @@ -26,7 +26,7 @@ import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; -import org.neo4j.gds.api.schema.NodeSchemaEntry; +import org.neo4j.gds.api.schema.MutableNodeSchemaEntry; import org.neo4j.gds.api.schema.PropertySchema; import java.io.IOException; @@ -61,7 +61,7 @@ void shouldLoadNodeSchemaCorrectly() throws IOException { var labelAProperties = nodeSchema.get(NodeLabel.of("A")); assertThat(labelAProperties) - .isEqualTo(new NodeSchemaEntry( + .isEqualTo(new MutableNodeSchemaEntry( NodeLabel.of("A"), Map.of( "prop1", @@ -76,7 +76,7 @@ void shouldLoadNodeSchemaCorrectly() throws IOException { var labelBProperties = nodeSchema.get(NodeLabel.of("B")); assertThat(labelBProperties) - .isEqualTo(new NodeSchemaEntry( + .isEqualTo(new MutableNodeSchemaEntry( NodeLabel.of("B"), Map.of( "prop2", @@ -126,7 +126,7 @@ void shouldLoadMixedLabels() throws IOException { var labelAProperties = nodeSchema.get(NodeLabel.of("A")); assertThat(labelAProperties) - .isEqualTo(new NodeSchemaEntry( + .isEqualTo(new MutableNodeSchemaEntry( NodeLabel.of("A"), Map.of( "prop1", @@ -141,7 +141,7 @@ void shouldLoadMixedLabels() throws IOException { var labelBProperties = nodeSchema.get(NodeLabel.of("B")); assertThat(labelBProperties) - .isEqualTo(new NodeSchemaEntry(NodeLabel.of("B"), Map.of())); + .isEqualTo(new MutableNodeSchemaEntry(NodeLabel.of("B"), Map.of())); } } diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoaderTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoaderTest.java index 303daa5dbd1..345c13a0392 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoaderTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/RelationshipSchemaLoaderTest.java @@ -27,8 +27,8 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import org.neo4j.gds.core.Aggregation; import java.io.IOException; @@ -59,7 +59,7 @@ void shouldLoadRelationshipSchemaCorrectly() throws IOException { var rel1Properties = loadedRelationshipSchema.get(RelationshipType.of("REL1")); assertThat(rel1Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL1"), Direction.DIRECTED, Map.of( @@ -76,7 +76,7 @@ void shouldLoadRelationshipSchemaCorrectly() throws IOException { var rel2Properties = loadedRelationshipSchema.get(RelationshipType.of("REL2")); assertThat(rel2Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL2"), Direction.UNDIRECTED, Map.of( @@ -125,7 +125,7 @@ void shouldLoadMixedRelationshipSchema() throws IOException { var rel1Properties = loadedRelationshipSchema.get(RelationshipType.of("REL1")); assertThat(rel1Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL1"), Direction.DIRECTED, Map.of( @@ -142,7 +142,7 @@ void shouldLoadMixedRelationshipSchema() throws IOException { var rel3Properties = loadedRelationshipSchema.get(RelationshipType.of("REL3")); assertThat(rel3Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL3"), Direction.UNDIRECTED, Map.of() @@ -166,7 +166,7 @@ void shouldLoadRelSchemaWithoutOrientation() throws IOException { var rel1Properties = loadedRelationshipSchema.get(RelationshipType.of("REL1")); assertThat(rel1Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL1"), Direction.DIRECTED, Map.of( @@ -183,7 +183,7 @@ void shouldLoadRelSchemaWithoutOrientation() throws IOException { var rel3Properties = loadedRelationshipSchema.get(RelationshipType.of("REL3")); assertThat(rel3Properties) - .isEqualTo(new RelationshipSchemaEntry( + .isEqualTo(new MutableRelationshipSchemaEntry( RelationshipType.of("REL3"), Direction.DIRECTED, Map.of() diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationToModelConverterTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationToModelConverterTest.java index 428f50feae4..8beb363b5bd 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationToModelConverterTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationToModelConverterTest.java @@ -25,14 +25,14 @@ import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.PropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.collections.LongMultiSet; import org.neo4j.gds.ml.core.subgraph.LocalIdMap; -import org.neo4j.gds.ml.metrics.ModelCandidateStats; import org.neo4j.gds.ml.metrics.EvaluationScores; +import org.neo4j.gds.ml.metrics.ModelCandidateStats; import org.neo4j.gds.ml.metrics.classification.ClassificationMetricSpecification; import org.neo4j.gds.ml.models.logisticregression.LogisticRegressionClassifier; import org.neo4j.gds.ml.models.logisticregression.LogisticRegressionData; @@ -94,9 +94,10 @@ void convertsModel() { .build(); var converter = new NodeClassificationToModelConverter(pipeline, config); - var originalSchema = ImmutableGraphSchema.builder() - .nodeSchema(NodeSchema.empty().addLabel(NodeLabel.of("M"))) - .relationshipSchema(RelationshipSchema.empty().addRelationshipType(RelationshipType.of("R"), Direction.UNDIRECTED)) + var originalSchema = MutableGraphSchema.builder() + .nodeSchema(MutableNodeSchema.empty().addLabel(NodeLabel.of("M"))) + .relationshipSchema(MutableRelationshipSchema + .empty().addRelationshipType(RelationshipType.of("R"), Direction.UNDIRECTED)) .putGraphProperty("array", PropertySchema.of("array", ValueType.DOUBLE_ARRAY)) .putGraphProperty("scalar", PropertySchema.of("scalar", ValueType.DOUBLE)) .build(); diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java index be3b39d2d29..0dafb6afcbd 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/GraphStoreFilter.java @@ -27,7 +27,9 @@ import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.beta.filter.expression.Expression; import org.neo4j.gds.beta.filter.expression.ExpressionParser; import org.neo4j.gds.beta.filter.expression.SemanticErrors; @@ -173,20 +175,22 @@ private static String replaceStarWithTrue(String filter) { return filter.equals(ElementProjection.PROJECT_ALL) ? "true" : filter; } - public static GraphSchema filterSchema(GraphSchema inputGraphSchema, NodesFilter.FilteredNodes filteredNodes, Set filteredRelationshipTypes) { - var nodeSchema = inputGraphSchema.nodeSchema().filter(filteredNodes.idMap().availableNodeLabels()); + public static MutableGraphSchema filterSchema(GraphSchema inputGraphSchema, NodesFilter.FilteredNodes filteredNodes, Set filteredRelationshipTypes) { + var nodeSchema = MutableNodeSchema.from(inputGraphSchema.nodeSchema().filter(filteredNodes.idMap().availableNodeLabels())); if (nodeSchema.availableLabels().isEmpty()) { nodeSchema.addLabel(NodeLabel.ALL_NODES); } - RelationshipSchema relationshipSchema = inputGraphSchema - .relationshipSchema() - .filter(filteredRelationshipTypes); + var relationshipSchema = MutableRelationshipSchema.from( + inputGraphSchema + .relationshipSchema() + .filter(filteredRelationshipTypes) + ); if (relationshipSchema.availableTypes().isEmpty()) { relationshipSchema.addRelationshipType(RelationshipType.ALL_RELATIONSHIPS, Direction.DIRECTED); } - return GraphSchema.of(nodeSchema, relationshipSchema, Map.of()); + return MutableGraphSchema.of(nodeSchema, relationshipSchema, Map.of()); } private GraphStoreFilter() {} diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index dee8d5333f9..69f751286e8 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -31,8 +31,8 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.GraphSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; import org.neo4j.gds.core.Username; @@ -161,18 +161,18 @@ protected ProgressTracker initProgressTracker() { } @Override - protected GraphSchema computeGraphSchema(Nodes nodes, RelationshipImportResult relationshipImportResult) { + protected MutableGraphSchema computeGraphSchema(Nodes nodes, RelationshipImportResult relationshipImportResult) { var relationshipSchema = relationshipImportResult.importResults().entrySet().stream().reduce( - RelationshipSchema.empty(), + MutableRelationshipSchema.empty(), (unionSchema, entry) -> { var relationshipType = entry.getKey(); var relationships = entry.getValue(); return unionSchema.union(relationships.relationshipSchema(relationshipType)); }, - RelationshipSchema::union + MutableRelationshipSchema::union ); - return GraphSchema.of( + return MutableGraphSchema.of( nodes.schema(), relationshipSchema, Map.of() @@ -284,7 +284,7 @@ private HashMap> propertyKeysByRelType() { var propertyKeysByRelType = new HashMap>(); Direction orientation = Direction.fromOrientation(graphProjectConfig.orientation()); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); gdlHandler.getEdges().forEach(edge -> { var relType = RelationshipType.of(edge.getLabel()); var entry = relationshipSchema.getOrCreateRelationshipType(relType, orientation); diff --git a/test-utils/src/test/java/org/neo4j/gds/gdl/GdlFactoryTest.java b/test-utils/src/test/java/org/neo4j/gds/gdl/GdlFactoryTest.java index bef3c9e5c99..7c4be3c5fea 100644 --- a/test-utils/src/test/java/org/neo4j/gds/gdl/GdlFactoryTest.java +++ b/test-utils/src/test/java/org/neo4j/gds/gdl/GdlFactoryTest.java @@ -32,8 +32,8 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.loading.CollectingConsumer; import java.util.ArrayList; @@ -262,7 +262,7 @@ void correctSchema() { ); var nodeSchema = graph.schema().nodeSchema(); - var expectedNodeSchema = NodeSchema.empty(); + var expectedNodeSchema = MutableNodeSchema.empty(); expectedNodeSchema.getOrCreateLabel(NodeLabel.of("A")) .addProperty("double", ValueType.DOUBLE) @@ -278,7 +278,7 @@ void correctSchema() { assertThat(nodeSchema).isEqualTo(expectedNodeSchema); var relationshipSchema = graph.schema().relationshipSchema(); - var expectedRelationshipSchema = RelationshipSchema.empty(); + var expectedRelationshipSchema = MutableRelationshipSchema.empty(); expectedRelationshipSchema.getOrCreateRelationshipType(RelationshipType.of("A"), Direction.DIRECTED) .addProperty("prop1", ValueType.DOUBLE, PropertyState.PERSISTENT) @@ -315,7 +315,7 @@ void correctRelationshipSchemaDirection(Orientation orientation) { .build() ).build().build(); - var expectedRelationshipSchema = RelationshipSchema.empty(); + var expectedRelationshipSchema = MutableRelationshipSchema.empty(); var direction = Direction.fromOrientation(orientation); expectedRelationshipSchema.getOrCreateRelationshipType(RelationshipType.of("A"), direction) From 91ff36637255b6a1b4879ba6fc566afa9bf464bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 16:34:52 +0100 Subject: [PATCH 138/400] Recreate hashCode and equals for MutableRelationshipSchema/Entry --- .../api/schema/MutableRelationshipSchema.java | 15 ++++++++++++++ .../MutableRelationshipSchemaEntry.java | 20 +++++++++++++++++++ .../api/schema/RelationshipSchemaTest.java | 4 ---- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java index b04ace26103..3f5d12f4e91 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchema.java @@ -156,4 +156,19 @@ public MutableRelationshipSchema addProperty( getOrCreateRelationshipType(relationshipType, direction).addProperty(propertyKey, valueType, propertyState); return this; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MutableRelationshipSchema that = (MutableRelationshipSchema) o; + + return entries.equals(that.entries); + } + + @Override + public int hashCode() { + return entries.hashCode(); + } } diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java index cd479283748..2f6826f2c63 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java @@ -131,4 +131,24 @@ public MutableRelationshipSchemaEntry addProperty(String propertyKey, Relationsh this.properties.put(propertyKey, propertySchema); return this; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MutableRelationshipSchemaEntry that = (MutableRelationshipSchemaEntry) o; + + if (!relationshipType.equals(that.relationshipType)) return false; + if (direction != that.direction) return false; + return properties.equals(that.properties); + } + + @Override + public int hashCode() { + int result = relationshipType.hashCode(); + result = 31 * result + direction.hashCode(); + result = 31 * result + properties.hashCode(); + return result; + } } diff --git a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java index 8b62f646777..54fce9de052 100644 --- a/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java +++ b/graph-schema-api/src/test/java/org/neo4j/gds/api/schema/RelationshipSchemaTest.java @@ -44,10 +44,6 @@ void inAndOutUndirected() { assertThat(MutableRelationshipSchema.empty().isUndirected()).isTrue(); RelationshipType type = RelationshipType.of("TYPE"); - var propertySchema = Map.of( - type, - Map.of("prop", RelationshipPropertySchema.of("prop", ValueType.DOUBLE)) - ); var undirectedSchema = MutableRelationshipSchema.empty(); undirectedSchema.getOrCreateRelationshipType(type, Direction.UNDIRECTED); From c3159f8354e589186c1230fc763892dd821d05e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Tue, 24 Jan 2023 16:35:17 +0100 Subject: [PATCH 139/400] Bring back accidentally removed method in ElementSchema --- .../main/java/org/neo4j/gds/api/schema/ElementSchema.java | 7 +++---- .../src/main/java/org/neo4j/gds/api/schema/NodeSchema.java | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java index e96981be27e..0d5996c5843 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/ElementSchema.java @@ -54,10 +54,9 @@ default Set allProperties() { } default Set allProperties(ELEMENT_IDENTIFIER elementIdentifier) { - return entries() - .stream() - .flatMap(entry -> entry.properties().keySet().stream()) - .collect(Collectors.toSet()); + return Optional.ofNullable(get(elementIdentifier)) + .map(entry -> entry.properties().keySet()) + .orElse(Set.of()); } default boolean hasProperties() { diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java index c98fc9468d6..337395b2ccb 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/NodeSchema.java @@ -24,7 +24,7 @@ import java.util.Set; public interface NodeSchema extends ElementSchema { - Set availableLabels() ; + Set availableLabels(); boolean containsOnlyAllNodesLabel(); From 72bea3730267ce46f9c51921a314dea6783475c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 15:36:58 +0100 Subject: [PATCH 140/400] Address PR comments Co-authored-by: Martin Junghanns --- .../graphsage/algo/GraphSageAlgorithmFactoryTest.java | 10 +++++----- .../java/org/neo4j/gds/config/WriteConfigTest.java | 4 ++-- .../java/org/neo4j/gds/doc/ModelCatalogDocTest.java | 4 ++-- .../java/org/neo4j/gds/api/schema/GraphSchema.java | 8 ++++++++ .../org/neo4j/gds/api/schema/MutableGraphSchema.java | 3 +-- .../org/neo4j/gds/api/schema/MutableNodeSchema.java | 1 + .../LinkPredictionTrainingPipelineTest.java | 6 +++--- .../embeddings/graphsage/GraphSageTrainProcTest.java | 4 ++-- .../predict/LinkPredictionPipelineProcTestBase.java | 4 ++-- .../LinkPredictionPredictPipelineExecutorTest.java | 4 ++-- .../NodeClassificationPipelinePredictProcTestUtil.java | 4 ++-- .../NodeClassificationPredictPipelineExecutorTest.java | 4 ++-- .../predict/NodeRegressionModelTestUtil.java | 4 ++-- 13 files changed, 34 insertions(+), 26 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactoryTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactoryTest.java index 546f749c02a..1b065966c3a 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactoryTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageAlgorithmFactoryTest.java @@ -27,7 +27,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.model.InjectModelCatalog; @@ -279,7 +279,7 @@ void memoryEstimationTreeStructure() { var model = Model.of( "graphSage", - MutableGraphSchema.empty(), + GraphSchema.empty(), ModelData.of(new Layer[]{}, new SingleLabelFeatureFunction()), trainConfig, GraphSageModelTrainer.GraphSageTrainMetrics.empty() @@ -335,7 +335,7 @@ void memoryEstimationMutateTreeStructure() { var model = Model.of( "graphSage", - MutableGraphSchema.empty(), + GraphSchema.empty(), ModelData.of(new Layer[]{}, new SingleLabelFeatureFunction()), trainConfig, GraphSageModelTrainer.GraphSageTrainMetrics.empty() @@ -508,7 +508,7 @@ public String toString() { var model = Model.of( "graphSage", - MutableGraphSchema.empty(), + GraphSchema.empty(), ModelData.of(new Layer[]{}, new SingleLabelFeatureFunction()), trainConfig, GraphSageModelTrainer.GraphSageTrainMetrics.empty() @@ -555,7 +555,7 @@ void mutateHasPersistentPart() { var model = Model.of( "graphSage", - MutableGraphSchema.empty(), + GraphSchema.empty(), ModelData.of(new Layer[]{}, new SingleLabelFeatureFunction()), trainConfig, GraphSageModelTrainer.GraphSageTrainMetrics.empty() diff --git a/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java b/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java index 28847e38213..d0de4f6b2eb 100644 --- a/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java +++ b/core/src/test/java/org/neo4j/gds/config/WriteConfigTest.java @@ -23,7 +23,7 @@ import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.huge.DirectIdMap; @@ -53,7 +53,7 @@ void validateGraphStoreCapabilities(boolean isBackedByDatabase) { var testGraphStore = new GraphStoreBuilder() .databaseId(DatabaseId.from("neo4j")) .capabilities(ImmutableStaticCapabilities.of(isBackedByDatabase)) - .schema(MutableGraphSchema.empty()) + .schema(GraphSchema.mutable()) .nodes(nodes) .relationshipImportResult(RelationshipImportResult.of(Map.of())) .concurrency(1) diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/ModelCatalogDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/ModelCatalogDocTest.java index 8a29d3b80dc..01579b8b68b 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/ModelCatalogDocTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/ModelCatalogDocTest.java @@ -21,7 +21,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.embeddings.graphsage.GraphSageModelTrainer; @@ -43,7 +43,7 @@ abstract class ModelCatalogDocTest extends SingleFileDocTestBase { void loadModel() { modelCatalog.set(Model.of( GraphSage.MODEL_TYPE, - MutableGraphSchema.empty(), + GraphSchema.empty(), ModelData.of(new Layer[0], new SingleLabelFeatureFunction()), ImmutableGraphSageTrainConfig .builder() diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java index b1bf9601faf..4d0059f5ace 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/GraphSchema.java @@ -71,6 +71,14 @@ default Direction direction() { return relationshipSchema().isUndirected() ? Direction.UNDIRECTED : Direction.DIRECTED; } + static GraphSchema empty() { + return MutableGraphSchema.empty(); + } + + static MutableGraphSchema mutable() { + return MutableGraphSchema.empty(); + } + static String forPropertySchema(PS propertySchema) { if (propertySchema instanceof RelationshipPropertySchema) { return String.format( diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java index 88dec2644ec..ff5093ada1d 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableGraphSchema.java @@ -31,8 +31,7 @@ import java.util.stream.Stream; @ValueClass -public -interface MutableGraphSchema extends GraphSchema { +public interface MutableGraphSchema extends GraphSchema { @Override MutableNodeSchema nodeSchema(); diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java index 898af1e52b5..af9603e67a0 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableNodeSchema.java @@ -87,6 +87,7 @@ public MutableNodeSchemaEntry get(NodeLabel identifier) { public void set(MutableNodeSchemaEntry entry) { entries.put(entry.identifier(), entry); } + public void remove(NodeLabel identifier) { entries.remove(identifier); } diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java index 79443714c44..2fabcd53096 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/LinkPredictionTrainingPipelineTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.model.OpenModelCatalog; @@ -191,7 +191,7 @@ void deriveRelationshipWeightPropertyFromTrainedModel() { String modelName = "myModel"; modelCatalog.set(Model.of( "myAlgo", - MutableGraphSchema.empty(), + GraphSchema.empty(), 1L, TestWeightedTrainConfigImpl.builder() .modelUser("") @@ -223,7 +223,7 @@ void notDerivePropertyFromUnweightedTrainedModel() { String modelName = "myModel"; modelCatalog.set(Model.of( "myAlgo", - MutableGraphSchema.empty(), + GraphSchema.empty(), 1L, TestTrainConfigImpl.builder() .modelUser("") diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java index d80a8e590d8..f4251641d00 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTrainProcTest.java @@ -27,7 +27,7 @@ import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.TestProcedureRunner; import org.neo4j.gds.api.DatabaseId; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.core.loading.CSRGraphStore; import org.neo4j.gds.core.loading.GraphStoreCatalog; @@ -233,7 +233,7 @@ void shouldValidateModelBeforeTraining() { ); var model = Model.of( GraphSage.MODEL_TYPE, - MutableGraphSchema.empty(), + GraphSchema.empty(), 42, config, GraphSageModelTrainer.GraphSageTrainMetrics.empty() diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineProcTestBase.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineProcTestBase.java index fa7f4e2fee5..8b3837802d1 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineProcTestBase.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineProcTestBase.java @@ -25,7 +25,7 @@ import org.neo4j.gds.GdsCypher; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.DefaultValue; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.catalog.GraphListProc; import org.neo4j.gds.catalog.GraphProjectProc; import org.neo4j.gds.core.model.Model; @@ -103,7 +103,7 @@ private void withModelInCatalog() { modelCatalog.set(Model.of( MODEL_TYPE, - MutableGraphSchema.empty(), + GraphSchema.empty(), modelData, LinkPredictionTrainConfigImpl.builder() .modelUser(getUsername()) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java index 910189fd3f0..84108b351ac 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPredictPipelineExecutorTest.java @@ -26,7 +26,7 @@ import org.neo4j.gds.Orientation; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; @@ -355,7 +355,7 @@ void progressTracking() { Model.of( MODEL_TYPE, - MutableGraphSchema.empty(), + GraphSchema.empty(), modelData, LinkPredictionTrainConfigImpl.builder() .modelUser(username) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelinePredictProcTestUtil.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelinePredictProcTestUtil.java index 2b3d62791de..e45fab90736 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelinePredictProcTestUtil.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelinePredictProcTestUtil.java @@ -21,7 +21,7 @@ import org.apache.commons.lang3.mutable.MutableDouble; import org.junit.jupiter.params.provider.Arguments; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.core.model.ModelCatalog; import org.neo4j.gds.core.utils.mem.MemoryRange; @@ -100,7 +100,7 @@ static Model model = Model.of( NodeClassificationTrainingPipeline.MODEL_TYPE, - MutableGraphSchema.empty(), + GraphSchema.empty(), modelData, NodeClassificationPipelineTrainConfigImpl.builder() .modelUser(getUsername()) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionModelTestUtil.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionModelTestUtil.java index d477cc1a393..1d9d74f4b80 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionModelTestUtil.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionModelTestUtil.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.ml.pipeline.node.regression.predict; -import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.core.model.Model; import org.neo4j.gds.ml.core.functions.Weights; import org.neo4j.gds.ml.core.tensor.Matrix; @@ -53,7 +53,7 @@ static Model Date: Thu, 2 Feb 2023 14:59:50 +0100 Subject: [PATCH 141/400] Bump to Neo4j 4.4.17 --- README.adoc | 2 +- gradle/dependencies.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index 12e820c7d5b..a3348eb72fa 100644 --- a/README.adoc +++ b/README.adoc @@ -84,7 +84,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.3.0 |Neo4j 5.4.0 .5+<.^|GDS 2.3.x -|Neo4j 4.4.9 - 4.4.16 +|Neo4j 4.4.9 - 4.4.17 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 2687d587310..f76dfdc8378 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4' : properties.getOrDefault('neo4jVersion44', '4.4.16'), + '4.4' : properties.getOrDefault('neo4jVersion44', '4.4.17'), '5.1' : properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2' : properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3' : properties.getOrDefault('neo4jVersion53', '5.3.0'), From 3b12953e7eaefe3c4bfe3281dabd19224a0fd68c Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 3 Feb 2023 10:33:03 +0100 Subject: [PATCH 142/400] upd to gradle --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 131fb7fd0a7..94f9fab04cd 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.1' - gdsAuraVersion = '10' + gdsAuraVersion = '11' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From a0574be520080dd1add7252afd07e4cdf01e2305 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 31 Jan 2023 12:45:43 +0000 Subject: [PATCH 143/400] Make the cluster writes error Neo4j version dependent --- .../main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java | 5 +++++ .../java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java | 5 +++++ .../java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java | 5 +++++ .../java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java | 5 +++++ .../java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java | 5 +++++ .../src/main/java/org/neo4j/gds/compat/SettingProxyApi.java | 2 ++ .../src/main/java/org/neo4j/gds/compat/SettingProxy.java | 4 ++++ .../java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java | 2 +- .../org/neo4j/gds/preconditions/ClusterRestrictions.java | 2 +- 9 files changed, 33 insertions(+), 2 deletions(-) diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java index 88226c3dbf0..ac658882bf2 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/SettingProxyImpl.java @@ -77,4 +77,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab } config.set(GraphDatabaseSettings.mode, mode); } + + @Override + public String secondaryModeName() { + return "Read Replica"; + } } diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java index 97dbcff99c8..1412919356e 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/SettingProxyImpl.java @@ -79,4 +79,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab throw new RuntimeException("Could not get the permissions to set the mode field.", e); } } + + @Override + public String secondaryModeName() { + return "Secondary"; + } } diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java index ef92ef88db5..7dd4139a1e1 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/SettingProxyImpl.java @@ -79,4 +79,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab throw new RuntimeException("Could not get the permissions to set the mode field.", e); } } + + @Override + public String secondaryModeName() { + return "Secondary"; + } } diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java index abdaa7629e9..d1c06a9af61 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/SettingProxyImpl.java @@ -79,4 +79,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab throw new RuntimeException("Could not get the permissions to set the mode field.", e); } } + + @Override + public String secondaryModeName() { + return "Secondary"; + } } diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java index d21ef5f3eb8..c854e5a5a80 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/SettingProxyImpl.java @@ -79,4 +79,9 @@ public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatab throw new RuntimeException("Could not get the permissions to set the mode field.", e); } } + + @Override + public String secondaryModeName() { + return "Secondary"; + } } diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxyApi.java index 8e044460110..831441d08f6 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxyApi.java @@ -31,4 +31,6 @@ public interface SettingProxyApi { @TestOnly void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService); + + String secondaryModeName(); } diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxy.java index f046a796697..73e83753ea4 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/SettingProxy.java @@ -49,5 +49,9 @@ public static void setDatabaseMode(Config config, DatabaseMode databaseMode, Gra IMPL.setDatabaseMode(config, databaseMode, databaseService); } + public static String secondaryModeName() { + return IMPL.secondaryModeName(); + } + private SettingProxy() {} } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java index fd61d1afb38..d1277f20d47 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStoreExportProcTest.java @@ -344,7 +344,7 @@ void failsDatabaseExportWhenRunningOnCluster() { .hasMessageContaining("The requested operation") .hasMessageContaining("(Export a graph to Neo4j database)") .hasMessageContaining( - "is not available while running Neo4j Graph Data Science library on a Neo4j Causal Cluster." + "is not available while running Neo4j Graph Data Science library on a Neo4j Cluster." ); } diff --git a/proc/common/src/main/java/org/neo4j/gds/preconditions/ClusterRestrictions.java b/proc/common/src/main/java/org/neo4j/gds/preconditions/ClusterRestrictions.java index bbe5cbc86fc..a3cbd17f6eb 100644 --- a/proc/common/src/main/java/org/neo4j/gds/preconditions/ClusterRestrictions.java +++ b/proc/common/src/main/java/org/neo4j/gds/preconditions/ClusterRestrictions.java @@ -35,7 +35,7 @@ public static void disallowRunningOnCluster(GraphDatabaseService databaseService if (neo4jMode != DatabaseMode.SINGLE) { throw new IllegalStateException( "The requested operation (" + detail + - ") is not available while running Neo4j Graph Data Science library on a Neo4j Causal Cluster."); + ") is not available while running Neo4j Graph Data Science library on a Neo4j Cluster."); } } } From a108a628a76bbe5e9b06bd36c3d5186beb71968f Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 3 Feb 2023 16:42:55 +0100 Subject: [PATCH 144/400] Remove trailing comma in json block --- doc/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/package.json b/doc/package.json index fb566faf308..8ef75e66def 100644 --- a/doc/package.json +++ b/doc/package.json @@ -8,7 +8,7 @@ "start": "nodemon -e adoc --exec \"npm run build && npm run serve\"", "serve": "node server.js", "build": "antora preview.yml --stacktrace --log-format=pretty", - "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json", + "build-verify": "antora --stacktrace --fetch preview.yml --log-format=json --log-level=info --log-file ./build/log/log.json" }, "keywords": [ "antora", From 0062806c86f0effce11d9948b2dfa440b11abeb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Mon, 6 Feb 2023 13:03:25 +0100 Subject: [PATCH 145/400] Move storage engine adapter projects --- .../4.4/storage-engine-adapter/build.gradle | 0 .../_44/InMemoryCommandCreationContextImpl.java | 0 .../neo4j/gds/compat/_44/InMemoryCountsStore.java | 0 .../compat/_44/InMemoryMetaDataProviderImpl.java | 0 .../neo4j/gds/compat/_44/InMemoryNodeCursor.java | 0 .../gds/compat/_44/InMemoryNodePropertyCursor.java | 0 .../gds/compat/_44/InMemoryPropertyCursor.java | 0 .../compat/_44/InMemoryPropertySelectionImpl.java | 0 .../_44/InMemoryRelationshipPropertyCursor.java | 0 .../compat/_44/InMemoryRelationshipScanCursor.java | 0 .../_44/InMemoryRelationshipTraversalCursor.java | 0 .../compat/_44/InMemoryStorageEngineFactory.java | 0 .../gds/compat/_44/InMemoryStorageEngineImpl.java | 0 .../gds/compat/_44/InMemoryStorageLocksImpl.java | 0 .../neo4j/gds/compat/_44/InMemoryStoreVersion.java | 0 .../compat/_44/InMemoryTransactionIdStoreImpl.java | 0 .../neo4j/gds/compat/_44/InMemoryVersionCheck.java | 0 .../compat/_44/StorageEngineProxyFactoryImpl.java | 0 .../gds/compat/_44/StorageEngineProxyImpl.java | 0 .../InMemoryLogVersionRepository44.java | 0 .../InMemoryStorageCommandReaderFactory44.java | 0 .../recordstorage/InMemoryStorageReader44.java | 0 .../5.1/storage-engine-adapter/build.gradle | 0 .../compat/_51/StorageEngineProxyFactoryImpl.java | 0 .../_51/InMemoryCommandCreationContextImpl.java | 0 .../gds/compat/_51/InMemoryCountsStoreImpl.java | 0 .../compat/_51/InMemoryMetaDataProviderImpl.java | 0 .../neo4j/gds/compat/_51/InMemoryNodeCursor.java | 0 .../gds/compat/_51/InMemoryNodePropertyCursor.java | 0 .../gds/compat/_51/InMemoryPropertyCursor.java | 0 .../compat/_51/InMemoryPropertySelectionImpl.java | 0 .../_51/InMemoryRelationshipPropertyCursor.java | 0 .../compat/_51/InMemoryRelationshipScanCursor.java | 0 .../_51/InMemoryRelationshipTraversalCursor.java | 0 .../compat/_51/InMemoryStorageEngineFactory.java | 0 .../gds/compat/_51/InMemoryStorageEngineImpl.java | 0 .../gds/compat/_51/InMemoryStorageLocksImpl.java | 0 .../neo4j/gds/compat/_51/InMemoryStoreVersion.java | 0 .../compat/_51/InMemoryTransactionIdStoreImpl.java | 0 .../neo4j/gds/compat/_51/InMemoryVersionCheck.java | 0 .../compat/_51/StorageEngineProxyFactoryImpl.java | 0 .../gds/compat/_51/StorageEngineProxyImpl.java | 0 .../InMemoryLogVersionRepository51.java | 0 .../InMemoryStorageCommandReaderFactory51.java | 0 .../recordstorage/InMemoryStorageReader51.java | 0 .../5.2/storage-engine-adapter/build.gradle | 0 .../compat/_52/StorageEngineProxyFactoryImpl.java | 0 .../_52/InMemoryCommandCreationContextImpl.java | 0 .../gds/compat/_52/InMemoryCountsStoreImpl.java | 0 .../compat/_52/InMemoryMetaDataProviderImpl.java | 0 .../neo4j/gds/compat/_52/InMemoryNodeCursor.java | 0 .../gds/compat/_52/InMemoryNodePropertyCursor.java | 0 .../gds/compat/_52/InMemoryPropertyCursor.java | 0 .../compat/_52/InMemoryPropertySelectionImpl.java | 0 .../_52/InMemoryRelationshipPropertyCursor.java | 0 .../compat/_52/InMemoryRelationshipScanCursor.java | 0 .../_52/InMemoryRelationshipTraversalCursor.java | 0 .../compat/_52/InMemoryStorageEngineFactory.java | 0 .../gds/compat/_52/InMemoryStorageEngineImpl.java | 0 .../gds/compat/_52/InMemoryStorageLocksImpl.java | 0 .../neo4j/gds/compat/_52/InMemoryStoreVersion.java | 0 .../compat/_52/InMemoryTransactionIdStoreImpl.java | 0 .../neo4j/gds/compat/_52/InMemoryVersionCheck.java | 0 .../compat/_52/StorageEngineProxyFactoryImpl.java | 0 .../gds/compat/_52/StorageEngineProxyImpl.java | 0 .../InMemoryLogVersionRepository52.java | 0 .../InMemoryStorageCommandReaderFactory52.java | 0 .../recordstorage/InMemoryStorageReader52.java | 0 .../5.3/storage-engine-adapter/build.gradle | 0 .../compat/_53/StorageEngineProxyFactoryImpl.java | 0 .../_53/InMemoryCommandCreationContextImpl.java | 0 .../gds/compat/_53/InMemoryCountsStoreImpl.java | 0 .../compat/_53/InMemoryMetaDataProviderImpl.java | 0 .../neo4j/gds/compat/_53/InMemoryNodeCursor.java | 0 .../gds/compat/_53/InMemoryNodePropertyCursor.java | 0 .../gds/compat/_53/InMemoryPropertyCursor.java | 0 .../compat/_53/InMemoryPropertySelectionImpl.java | 0 .../_53/InMemoryRelationshipPropertyCursor.java | 0 .../compat/_53/InMemoryRelationshipScanCursor.java | 0 .../_53/InMemoryRelationshipTraversalCursor.java | 0 .../compat/_53/InMemoryStorageEngineFactory.java | 0 .../gds/compat/_53/InMemoryStorageEngineImpl.java | 0 .../gds/compat/_53/InMemoryStorageLocksImpl.java | 0 .../neo4j/gds/compat/_53/InMemoryStoreVersion.java | 0 .../compat/_53/InMemoryTransactionIdStoreImpl.java | 0 .../neo4j/gds/compat/_53/InMemoryVersionCheck.java | 0 .../compat/_53/StorageEngineProxyFactoryImpl.java | 0 .../gds/compat/_53/StorageEngineProxyImpl.java | 0 .../InMemoryLogVersionRepository.java | 0 .../InMemoryStorageCommandReaderFactory.java | 0 .../recordstorage/InMemoryStorageReader53.java | 0 .../5.4/storage-engine-adapter/build.gradle | 0 .../compat/_54/StorageEngineProxyFactoryImpl.java | 0 .../_54/InMemoryCommandCreationContextImpl.java | 0 .../gds/compat/_54/InMemoryCountsStoreImpl.java | 0 .../compat/_54/InMemoryMetaDataProviderImpl.java | 0 .../neo4j/gds/compat/_54/InMemoryNodeCursor.java | 0 .../gds/compat/_54/InMemoryNodePropertyCursor.java | 0 .../gds/compat/_54/InMemoryPropertyCursor.java | 0 .../compat/_54/InMemoryPropertySelectionImpl.java | 0 .../_54/InMemoryRelationshipPropertyCursor.java | 0 .../compat/_54/InMemoryRelationshipScanCursor.java | 0 .../_54/InMemoryRelationshipTraversalCursor.java | 0 .../compat/_54/InMemoryStorageEngineFactory.java | 0 .../gds/compat/_54/InMemoryStorageEngineImpl.java | 0 .../gds/compat/_54/InMemoryStorageLocksImpl.java | 0 .../neo4j/gds/compat/_54/InMemoryStoreVersion.java | 0 .../compat/_54/InMemoryTransactionIdStoreImpl.java | 0 .../neo4j/gds/compat/_54/InMemoryVersionCheck.java | 0 .../compat/_54/StorageEngineProxyFactoryImpl.java | 0 .../gds/compat/_54/StorageEngineProxyImpl.java | 0 .../InMemoryLogVersionRepository.java | 0 .../InMemoryStorageCommandReaderFactory.java | 0 .../recordstorage/InMemoryStorageReader54.java | 0 .../api/storage-engine-adapter/build.gradle | 0 .../gds/compat/AbstractInMemoryNodeCursor.java | 0 .../compat/AbstractInMemoryNodePropertyCursor.java | 0 .../gds/compat/AbstractInMemoryPropertyCursor.java | 0 ...AbstractInMemoryRelationshipPropertyCursor.java | 0 ...bstractInMemoryRelationshipTraversalCursor.java | 0 .../gds/compat/InMemoryPropertySelection.java | 0 .../neo4j/gds/compat/StorageEngineProxyApi.java | 0 .../gds/compat/StorageEngineProxyFactory.java | 0 .../java/org/neo4j/gds/compat/TokenManager.java | 0 .../InMemoryDatabaseCreationCatalog.java | 0 .../storageengine/InMemoryRelationshipCursor.java | 0 .../InMemoryTransactionStateVisitor.java | 0 .../AbstractInMemoryAllRelationshipScan.java | 0 .../AbstractInMemoryRelationshipScanCursor.java | 0 .../recordstorage/AbstractTransactionIdStore.java | 0 .../internal/recordstorage/InMemoryNodeScan.java | 0 .../common/storage-engine-adapter/build.gradle | 0 .../org/neo4j/gds/compat/StorageEngineProxy.java | 0 .../gds/storageengine/InMemoryDatabaseCreator.java | 0 .../storageengine/InMemoryDbShutdownExtension.java | 0 settings.gradle | 14 +++++++------- 136 files changed, 7 insertions(+), 7 deletions(-) rename {cypher => compatibility}/4.4/storage-engine-adapter/build.gradle (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java (100%) rename {cypher => compatibility}/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/build.gradle (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java (100%) rename {cypher => compatibility}/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/build.gradle (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java (100%) rename {cypher => compatibility}/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/build.gradle (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java (100%) rename {cypher => compatibility}/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/build.gradle (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java (100%) rename {cypher => compatibility}/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/build.gradle (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java (100%) rename {cypher => compatibility}/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java (100%) rename {cypher => compatibility}/common/storage-engine-adapter/build.gradle (100%) rename {cypher => compatibility}/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java (100%) rename {cypher => compatibility}/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java (100%) rename {cypher => compatibility}/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java (100%) diff --git a/cypher/4.4/storage-engine-adapter/build.gradle b/compatibility/4.4/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/4.4/storage-engine-adapter/build.gradle rename to compatibility/4.4/storage-engine-adapter/build.gradle diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCommandCreationContextImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryCountsStore.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryMetaDataProviderImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodeCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryNodePropertyCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertyCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryPropertySelectionImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipScanCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageLocksImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStoreVersion.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryVersionCheck.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyFactoryImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/StorageEngineProxyImpl.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository44.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory44.java diff --git a/cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java similarity index 100% rename from cypher/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java rename to compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryStorageReader44.java diff --git a/cypher/5.1/storage-engine-adapter/build.gradle b/compatibility/5.1/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/5.1/storage-engine-adapter/build.gradle rename to compatibility/5.1/storage-engine-adapter/build.gradle diff --git a/cypher/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCommandCreationContextImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryCountsStoreImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryMetaDataProviderImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodeCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryNodePropertyCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertyCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryPropertySelectionImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipScanCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageLocksImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStoreVersion.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryVersionCheck.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/StorageEngineProxyImpl.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository51.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory51.java diff --git a/cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java similarity index 100% rename from cypher/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java rename to compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader51.java diff --git a/cypher/5.2/storage-engine-adapter/build.gradle b/compatibility/5.2/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/5.2/storage-engine-adapter/build.gradle rename to compatibility/5.2/storage-engine-adapter/build.gradle diff --git a/cypher/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCommandCreationContextImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryCountsStoreImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryMetaDataProviderImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodeCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryNodePropertyCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertyCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryPropertySelectionImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipScanCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageLocksImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStoreVersion.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryVersionCheck.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/StorageEngineProxyImpl.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository52.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory52.java diff --git a/cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java similarity index 100% rename from cypher/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java rename to compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader52.java diff --git a/cypher/5.3/storage-engine-adapter/build.gradle b/compatibility/5.3/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/5.3/storage-engine-adapter/build.gradle rename to compatibility/5.3/storage-engine-adapter/build.gradle diff --git a/cypher/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCommandCreationContextImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryCountsStoreImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodeCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryNodePropertyCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertyCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryPropertySelectionImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipScanCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageLocksImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStoreVersion.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryVersionCheck.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/StorageEngineProxyImpl.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java diff --git a/cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java similarity index 100% rename from cypher/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader53.java diff --git a/cypher/5.4/storage-engine-adapter/build.gradle b/compatibility/5.4/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/5.4/storage-engine-adapter/build.gradle rename to compatibility/5.4/storage-engine-adapter/build.gradle diff --git a/cypher/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCommandCreationContextImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryCountsStoreImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodeCursor.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryNodePropertyCursor.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertyCursor.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryPropertySelectionImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipPropertyCursor.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipScanCursor.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryRelationshipTraversalCursor.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageLocksImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStoreVersion.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryVersionCheck.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyFactoryImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/StorageEngineProxyImpl.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java diff --git a/cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java similarity index 100% rename from cypher/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader54.java diff --git a/cypher/api/storage-engine-adapter/build.gradle b/compatibility/api/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/api/storage-engine-adapter/build.gradle rename to compatibility/api/storage-engine-adapter/build.gradle diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodeCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryNodePropertyCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryPropertyCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipPropertyCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/AbstractInMemoryRelationshipTraversalCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/InMemoryPropertySelection.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyFactory.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/TokenManager.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreationCatalog.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryRelationshipCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryTransactionStateVisitor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryAllRelationshipScan.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractInMemoryRelationshipScanCursor.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java diff --git a/cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java similarity index 100% rename from cypher/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java rename to compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/InMemoryNodeScan.java diff --git a/cypher/common/storage-engine-adapter/build.gradle b/compatibility/common/storage-engine-adapter/build.gradle similarity index 100% rename from cypher/common/storage-engine-adapter/build.gradle rename to compatibility/common/storage-engine-adapter/build.gradle diff --git a/cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java similarity index 100% rename from cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java rename to compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxy.java diff --git a/cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java similarity index 100% rename from cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java rename to compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDatabaseCreator.java diff --git a/cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java b/compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java similarity index 100% rename from cypher/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java rename to compatibility/common/storage-engine-adapter/src/main/java/org/neo4j/gds/storageengine/InMemoryDbShutdownExtension.java diff --git a/settings.gradle b/settings.gradle index dfb58392ff8..83a43b9ec08 100644 --- a/settings.gradle +++ b/settings.gradle @@ -203,25 +203,25 @@ include('proc-pipeline-catalog') project(':proc-pipeline-catalog').projectDir = file('proc/pipeline-catalog') include('storage-engine-adapter') -project(':storage-engine-adapter').projectDir = file('cypher/common/storage-engine-adapter') +project(':storage-engine-adapter').projectDir = file('compatibility/common/storage-engine-adapter') include('storage-engine-adapter-4.4') -project(':storage-engine-adapter-4.4').projectDir = file('cypher/4.4/storage-engine-adapter') +project(':storage-engine-adapter-4.4').projectDir = file('compatibility/4.4/storage-engine-adapter') include('storage-engine-adapter-5.1') -project(':storage-engine-adapter-5.1').projectDir = file('cypher/5.1/storage-engine-adapter') +project(':storage-engine-adapter-5.1').projectDir = file('compatibility/5.1/storage-engine-adapter') include('storage-engine-adapter-5.2') -project(':storage-engine-adapter-5.2').projectDir = file('cypher/5.2/storage-engine-adapter') +project(':storage-engine-adapter-5.2').projectDir = file('compatibility/5.2/storage-engine-adapter') include('storage-engine-adapter-5.3') -project(':storage-engine-adapter-5.3').projectDir = file('cypher/5.3/storage-engine-adapter') +project(':storage-engine-adapter-5.3').projectDir = file('compatibility/5.3/storage-engine-adapter') include('storage-engine-adapter-5.4') -project(':storage-engine-adapter-5.4').projectDir = file('cypher/5.4/storage-engine-adapter') +project(':storage-engine-adapter-5.4').projectDir = file('compatibility/5.4/storage-engine-adapter') include('storage-engine-adapter-api') -project(':storage-engine-adapter-api').projectDir = file('cypher/api/storage-engine-adapter') +project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') include('string-formatting') project(':string-formatting').projectDir = file('string-formatting') From c147d655c41452457853548c4d68c8e86268e650 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Thu, 2 Feb 2023 12:05:47 +0100 Subject: [PATCH 146/400] Document tuning of random seed for HashGNN --- .../node-embeddings/hashgnn.adoc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc index 07c0d1a50de..c3e62b44e34 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc @@ -238,6 +238,22 @@ This is especially true for unsupervised embedding algorithms such as HashGNN. Therefore, caution should be taken when using many iterations in the heterogeneous mode. +=== Random seed + +The random seed has a special role in this algorithm. +Other than making all steps of the algorithm deterministic, the `randomSeed` parameter determines which (to some degree) hash functions are used inside the algorithm. +This is important since it greatly affects which features are sampled each iteration. +The hashing plays a similar role to the (typically neural) transformations in each layer of Graph Neural Networks, which tells us something about how important the hash functions are. +Indeed, one can often see a significant difference in the quality of the node embeddings output from the algorithm when only the `randomSeed` is different in the configuration. + +For these reasons it can actually make sense to tune the random seed parameter. +Note that it should be tuned as a categorical (i.e. non-ordinal) number, meaning that values 1 and 2 can be considered just as similar or different as 1 and 100. +A good way to start doing this is to choose 5 - 10 arbitrary integers (eg. values 1, 2, 3, 4 and 5) as the candidates for the random seed. + +`randomSeed` codepends on several configuration parameters, and in particular on the `neighborInfluence` parameter which also directly influences which hash functions are used. +Therefore, if `neighborInfluence` is changed, likely the `randomSeed` parameter needs to be retuned. + + [[algorithms-embeddings-hashgnn-syntax]] == Syntax From e40e9117c8596ed5541e85b3bdb8458bd9272e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Mon, 6 Feb 2023 17:09:17 +0100 Subject: [PATCH 147/400] Can load 5.5 compat Co-Authored-By: Yuval Rotenberg --- .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 9 ++++++--- .../src/test/java/org/neo4j/gds/SysInfoProcTest.java | 8 ++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 6dcf3a1d681..ba4140d6624 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -32,6 +32,7 @@ public enum Neo4jVersion { V_5_2, V_5_3, V_5_4, + V_5_5, V_RC; @Override @@ -47,8 +48,8 @@ public String toString() { return "5.3"; case V_5_4: return "5.4"; - case V_RC: - return "rc"; + case V_5_5: + return "5.5"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } @@ -56,7 +57,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 5); + return ImmutableMajorMinorVersion.of(5, 6); } String version = toString(); var subVersions = version.split("\\."); @@ -127,6 +128,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 4) { return Neo4jVersion.V_5_4; } else if (minorVersion == 5) { + return Neo4jVersion.V_5_5; + } else if (minorVersion == 6) { return Neo4jVersion.V_RC; } } diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 675e3a2a279..b43b34c7613 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -163,6 +163,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.4" ); break; + case V_5_5: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.5 (placeholder)", + "Neo4j Settings 5.5", + "Neo4j 5.5 (placeholder)", + "Neo4j 5.5" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC (placeholder)", From f8876966df659c7c2b7eda8fd26c1159d7ca30fa Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Tue, 7 Feb 2023 09:53:15 +0100 Subject: [PATCH 148/400] Add links to relevant client notebooks in docs --- doc/modules/ROOT/pages/algorithms/knn.adoc | 10 ++++++++-- .../ROOT/pages/machine-learning/auto-tuning.adoc | 4 ++++ .../machine-learning/node-embeddings/fastrp.adoc | 7 ++++++- .../machine-learning/node-embeddings/hashgnn.adoc | 5 +++++ .../node-classification.adoc | 12 ++++++++++++ .../noderegression-pipelines/node-regression.adoc | 4 ++++ 6 files changed, 39 insertions(+), 3 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/knn.adoc b/doc/modules/ROOT/pages/algorithms/knn.adoc index 67e684d49e6..b6d79843ece 100644 --- a/doc/modules/ROOT/pages/algorithms/knn.adoc +++ b/doc/modules/ROOT/pages/algorithms/knn.adoc @@ -7,11 +7,17 @@ :algorithm: K-Nearest Neighbors :knnSpecificConfigurationTableTitle: Algorithm specific configuration - :directed: :undirected: :homogeneous: :weighted: +include::partial$/algorithms/shared/algorithm-traits.adoc[] + +{nbsp} + +[NOTE] +==== +kNN is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example]. +==== [[algorithms-knn-intro]] == Introduction @@ -19,7 +25,7 @@ The K-Nearest Neighbors algorithm computes a distance value for all node pairs in the graph and creates new relationships between each node and its k nearest neighbors. The distance is calculated based on node properties. -The input of this algorithm is a monopartite graph. +The input of this algorithm is a homogeneous graph. The graph does not need to be connected, in fact, existing relationships between nodes will be ignored - apart from random walk sampling if that that initial sampling option is used. New relationships are created between each node and its k nearest neighbors. diff --git a/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc b/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc index 3ec6eb7e72c..cbcaa865ab9 100644 --- a/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc +++ b/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc @@ -2,6 +2,10 @@ = Auto-tuning :description: This section describes auto-tuning for hyper-parameters in training pipelines in the Neo4j Graph Data Science library. +[NOTE] +==== +Auto-tuning is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning]. +==== xref:machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc[Node Classification Pipelines], xref:machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc[Node Regression Pipelines], and xref:machine-learning/linkprediction-pipelines/link-prediction.adoc[Link Prediction Pipelines] are trained using supervised machine learning methods which have multiple configurable parameters that affect training outcomes. To obtain models with high quality, setting good values for the hyper-parameters can have a large impact. diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc index 6dc3b2c7256..69895f3cda5 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc @@ -5,13 +5,18 @@ :result: embedding :algorithm: FastRP - :directed: :undirected: :homogeneous: :weighted: include::partial$/algorithms/shared/algorithm-traits.adoc[] +{nbsp} + +[NOTE] +==== +FastRP is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example]. +==== + [[algorithms-embeddings-fastrp-introduction]] == Introduction diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc index c3e62b44e34..74023a778cb 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc @@ -12,6 +12,11 @@ include::partial$/operations-reference/beta-note.adoc[] :heterogeneous: include::partial$/algorithms/shared/algorithm-traits.adoc[] +{nbsp} + +[NOTE] +==== +HashGNN is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning]. +==== [[algorithms-embeddings-hashgnn-introduction]] == Introduction diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc index 50309dbd7e0..6428f3cfcf9 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc @@ -7,6 +7,13 @@ include::partial$/operations-reference/beta-note.adoc[] +[NOTE] +==== +Node classification pipelines are featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/ml-pipelines-node-classification.ipynb[Machine learning pipelines: Node classification] +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning] +==== Node Classification is a common machine learning task applied to graphs: training models to classify nodes. Concretely, Node Classification models are used to predict the classes of unlabeled nodes as a node properties based on other node properties. @@ -35,6 +42,11 @@ The order of the probabilities matches the order of the classes registered in th NOTE: xref:machine-learning/node-property-prediction/nodeclassification-pipelines/predict.adoc[Classification] can only be done with a classification model (not with a training pipeline). +The node classification pipeline is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning]. +* + This segment is divided into the following pages: * xref:machine-learning/node-property-prediction/nodeclassification-pipelines/config.adoc[Configuring the pipeline] diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc index 1d75fa50222..79d03bfd252 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc @@ -6,6 +6,10 @@ include::partial$/operations-reference/alpha-note.adoc[] +[NOTE] +==== +Node regression pipelines are featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression pipeline]. +==== Node Regression is a common machine learning task applied to graphs: training models to predict node property values. Concretely, Node Regression models are used to predict the value of node property based on other node properties. From 0fe52ebbd2ef1333e38ac29a32e0d4a4f5235870 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Tue, 7 Feb 2023 11:15:50 +0100 Subject: [PATCH 149/400] Remove duplicate references to notebooks in NC docs Co-Authored-By: Mats Rydberg --- .../nodeclassification-pipelines/node-classification.adoc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc index 6428f3cfcf9..e963ab59fe7 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc @@ -42,11 +42,6 @@ The order of the probabilities matches the order of the classes registered in th NOTE: xref:machine-learning/node-property-prediction/nodeclassification-pipelines/predict.adoc[Classification] can only be done with a classification model (not with a training pipeline). -The node classification pipeline is featured in the end-to-end example Jupyter notebooks: - -* https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning]. -* - This segment is divided into the following pages: * xref:machine-learning/node-property-prediction/nodeclassification-pipelines/config.adoc[Configuring the pipeline] From 18d6b8d97450d28181d967dacd49f3a62392556f Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Tue, 7 Feb 2023 11:20:24 +0100 Subject: [PATCH 150/400] Stylize notebook examples reference admonitions as TIPs Co-Authored-By: Mats Rydberg --- doc/modules/ROOT/pages/algorithms/knn.adoc | 2 +- doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc | 2 +- .../ROOT/pages/machine-learning/node-embeddings/fastrp.adoc | 2 +- .../ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc | 2 +- .../nodeclassification-pipelines/node-classification.adoc | 2 +- .../noderegression-pipelines/node-regression.adoc | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/knn.adoc b/doc/modules/ROOT/pages/algorithms/knn.adoc index b6d79843ece..706c821367e 100644 --- a/doc/modules/ROOT/pages/algorithms/knn.adoc +++ b/doc/modules/ROOT/pages/algorithms/knn.adoc @@ -14,7 +14,7 @@ include::partial$/algorithms/shared/algorithm-traits.adoc[] {nbsp} + -[NOTE] +[TIP] ==== kNN is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example]. ==== diff --git a/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc b/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc index cbcaa865ab9..d230fae7ba9 100644 --- a/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc +++ b/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc @@ -2,7 +2,7 @@ = Auto-tuning :description: This section describes auto-tuning for hyper-parameters in training pipelines in the Neo4j Graph Data Science library. -[NOTE] +[TIP] ==== Auto-tuning is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning]. ==== diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc index 69895f3cda5..d92876436ec 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc @@ -12,7 +12,7 @@ include::partial$/algorithms/shared/algorithm-traits.adoc[] {nbsp} + -[NOTE] +[TIP] ==== FastRP is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example]. ==== diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc index 74023a778cb..2eef295b84e 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc @@ -13,7 +13,7 @@ include::partial$/operations-reference/beta-note.adoc[] include::partial$/algorithms/shared/algorithm-traits.adoc[] {nbsp} + -[NOTE] +[TIP] ==== HashGNN is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning]. ==== diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc index e963ab59fe7..84af4a99b14 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc @@ -7,7 +7,7 @@ include::partial$/operations-reference/beta-note.adoc[] -[NOTE] +[TIP] ==== Node classification pipelines are featured in the end-to-end example Jupyter notebooks: diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc index 79d03bfd252..c045d1c9d78 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc @@ -6,7 +6,7 @@ include::partial$/operations-reference/alpha-note.adoc[] -[NOTE] +[TIP] ==== Node regression pipelines are featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression pipeline]. ==== From 126c7e293acc8499e4c9c8c7954f601dd1bd2967 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Tue, 7 Feb 2023 11:24:13 +0100 Subject: [PATCH 151/400] Make client notebook refences be listed as bullet point in docs Co-Authored-By: Mats Rydberg --- doc/modules/ROOT/pages/algorithms/knn.adoc | 4 +++- doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc | 4 +++- .../ROOT/pages/machine-learning/node-embeddings/fastrp.adoc | 4 +++- .../ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc | 4 +++- .../noderegression-pipelines/node-regression.adoc | 4 +++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/knn.adoc b/doc/modules/ROOT/pages/algorithms/knn.adoc index 706c821367e..5f5a6023c3e 100644 --- a/doc/modules/ROOT/pages/algorithms/knn.adoc +++ b/doc/modules/ROOT/pages/algorithms/knn.adoc @@ -16,7 +16,9 @@ include::partial$/algorithms/shared/algorithm-traits.adoc[] {nbsp} + [TIP] ==== -kNN is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example]. +kNN is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example] ==== [[algorithms-knn-intro]] diff --git a/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc b/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc index d230fae7ba9..d81ceec1a63 100644 --- a/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc +++ b/doc/modules/ROOT/pages/machine-learning/auto-tuning.adoc @@ -4,7 +4,9 @@ [TIP] ==== -Auto-tuning is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning]. +Auto-tuning is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning] ==== xref:machine-learning/node-property-prediction/nodeclassification-pipelines/node-classification.adoc[Node Classification Pipelines], xref:machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc[Node Regression Pipelines], and xref:machine-learning/linkprediction-pipelines/link-prediction.adoc[Link Prediction Pipelines] are trained using supervised machine learning methods which have multiple configurable parameters that affect training outcomes. diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc index d92876436ec..fb46b397873 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc @@ -14,7 +14,9 @@ include::partial$/algorithms/shared/algorithm-traits.adoc[] {nbsp} + [TIP] ==== -FastRP is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example]. +FastRP is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/fastrp-and-knn.ipynb[FastRP and kNN end-to-end example] ==== diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc index 2eef295b84e..beb25ddca08 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc @@ -15,7 +15,9 @@ include::partial$/algorithms/shared/algorithm-traits.adoc[] {nbsp} + [TIP] ==== -HashGNN is featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning]. +HashGNN is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/heterogeneous-node-classification-with-hashgnn.ipynb[Heterogeneous Node Classification with HashGNN and Autotuning] ==== [[algorithms-embeddings-hashgnn-introduction]] diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc index c045d1c9d78..7c31f41ddb1 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc @@ -8,7 +8,9 @@ include::partial$/operations-reference/alpha-note.adoc[] [TIP] ==== -Node regression pipelines are featured in the end-to-end example Jupyter notebook https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression pipeline]. +Node regression pipelines are featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression pipeline] ==== Node Regression is a common machine learning task applied to graphs: training models to predict node property values. From 3f25be34ad47b47e85ec9e68c971b2bb30201034 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Tue, 7 Feb 2023 11:35:50 +0100 Subject: [PATCH 152/400] Update title of NR notebook reference Co-Authored-By: Mats Rydberg --- .../noderegression-pipelines/node-regression.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc index 7c31f41ddb1..ecb96a378e0 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-property-prediction/noderegression-pipelines/node-regression.adoc @@ -10,7 +10,7 @@ include::partial$/operations-reference/alpha-note.adoc[] ==== Node regression pipelines are featured in the end-to-end example Jupyter notebooks: -* https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression pipeline] +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression with Subgraph and Graph Sample projections] ==== Node Regression is a common machine learning task applied to graphs: training models to predict node property values. From ba4404f2f4d9d35e7355705bf7de5568e24d82e0 Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Tue, 7 Feb 2023 11:48:54 +0100 Subject: [PATCH 153/400] Add notebook references in graph catalog docs Co-Authored-By: Mats Rydberg --- .../projections/graph-project-subgraph.adoc | 8 +++++++- .../ROOT/pages/management-ops/projections/rwr.adoc | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-subgraph.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-subgraph.adoc index a2c6514c01c..a96b9a7bea6 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-subgraph.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-subgraph.adoc @@ -3,9 +3,15 @@ = Projecting a subgraph :description: This section details how to project subgraphs from existing graphs stored in the graph catalog of the Neo4j Graph Data Science library. - include::partial$/operations-reference/beta-note.adoc[] +[TIP] +==== +Subgraph projection is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression with Subgraph and Graph Sample projections] +==== + In GDS, algorithms can be executed on a named graph that has been filtered based on its xref:common-usage/running-algos.adoc#common-configuration-node-labels[node labels] and xref:common-usage/running-algos.adoc#common-configuration-relationship-types[relationship types]. However, that filtered graph only exists during the execution of the algorithm, and it is not possible to filter on property values. If a filtered graph needs to be used multiple times, one can use the subgraph catalog procedure to project a new graph in the graph catalog. diff --git a/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc b/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc index 97dd211363f..6044fd9af4a 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc @@ -12,6 +12,14 @@ include::partial$/operations-reference/alpha-note.adoc[] :weighted: include::partial$/algorithms/shared/algorithm-traits.adoc[] +{nbsp} + +[TIP] +==== +Random walk with restarts sampling is featured in the end-to-end example Jupyter notebooks: + +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/import-sample-export-gnn.ipynb[Sampling, Export and Integration with PyG example] +* https://github.com/neo4j/graph-data-science-client/blob/main/examples/node-regression-with-subgraph-and-graph-sample.ipynb[Node Regression with Subgraph and Graph Sample projections] +==== == Introduction From 211c52f63459b933576c4c337ecb6383cb6e6ff0 Mon Sep 17 00:00:00 2001 From: Max Kiessling Date: Wed, 25 Jan 2023 21:36:13 +0100 Subject: [PATCH 154/400] Add RelationshipSchemaEntry to SingleTypeRelationships --- .../core/loading/SingleTypeRelationships.java | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java index 8991a5f5e37..df01f1ea263 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java @@ -30,6 +30,7 @@ import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.gds.api.schema.RelationshipPropertySchema; +import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import java.util.Optional; @@ -47,34 +48,14 @@ public interface SingleTypeRelationships { Topology topology(); + RelationshipSchemaEntry relationshipSchemaEntry(); + Optional properties(); Optional inverseTopology(); Optional inverseProperties(); - default MutableRelationshipSchema relationshipSchema(RelationshipType relationshipType) { - var schema = MutableRelationshipSchema.empty(); - this.updateRelationshipSchemaEntry(schema.getOrCreateRelationshipType(relationshipType, direction())); - return schema; - } - - default void updateRelationshipSchemaEntry(MutableRelationshipSchemaEntry schemaEntry) { - properties().ifPresent(relationshipPropertyStore -> relationshipPropertyStore - .relationshipProperties() - .forEach((propertyKey, relationshipProperty) -> { - schemaEntry.addProperty( - propertyKey, - RelationshipPropertySchema.of(propertyKey, - relationshipProperty.valueType(), - relationshipProperty.defaultValue(), - relationshipProperty.propertyState(), - relationshipProperty.aggregation() - ) - ); - })); - } - /** * Filters the relationships to include only the given property if present. */ @@ -86,8 +67,14 @@ default SingleTypeRelationships filter(String propertyKey) { relationshipPropertyStore.filter(propertyKey) ); + + RelationshipSchemaEntry entry = relationshipSchemaEntry(); + var filteredEntry = new RelationshipSchemaEntry(entry.identifier(), entry.direction()) + .addProperty(propertyKey, entry.properties().get(propertyKey)); + return SingleTypeRelationships.builder() .topology(topology()) + .relationshipSchemaEntry(filteredEntry) .direction(direction()) .inverseTopology(inverseTopology()) .properties(properties) @@ -111,14 +98,19 @@ static ImmutableSingleTypeRelationships.Builder builder() { } static SingleTypeRelationships of( + RelationshipType relationshipType, Topology topology, Direction direction, Optional properties, Optional propertySchema ) { + var schemaEntry = new RelationshipSchemaEntry(relationshipType, direction); + propertySchema.ifPresent(schema -> schemaEntry.addProperty(schema.key(), schema)); + return SingleTypeRelationships.builder() .direction(direction) .topology(topology) + .relationshipSchemaEntry(schemaEntry) .properties( propertySchema.map(schema -> { var relationshipProperty = ImmutableRelationshipProperty.builder() From bb454779d94141c957ac0414dfbd4a3204172c1a Mon Sep 17 00:00:00 2001 From: Max Kiessling Date: Wed, 25 Jan 2023 21:38:53 +0100 Subject: [PATCH 155/400] Use RelationshipSchemaEntry from SingleTypeRelationships --- .../neo4j/gds/beta/generator/RandomGraphGenerator.java | 6 +++++- .../java/org/neo4j/gds/core/loading/CSRGraphStore.java | 5 +---- .../gds/core/loading/construction/GraphFactory.java | 3 ++- .../java/org/neo4j/gds/projection/GraphAggregator.java | 10 ++++------ .../src/main/java/org/neo4j/gds/gdl/GdlFactory.java | 9 ++++----- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java index 74b7162a8fe..dc4f640ff9b 100644 --- a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java +++ b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java @@ -29,6 +29,7 @@ import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.config.RandomGraphGeneratorConfig.AllowSelfLoops; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.huge.HugeGraph; @@ -143,9 +144,12 @@ public HugeGraph generate() { var relationships = relationshipsBuilder.build(); + RelationshipSchema relationshipSchema = RelationshipSchema.empty(); + relationshipSchema.set(relationships.relationshipSchemaEntry()); + var graphSchema = MutableGraphSchema.of( nodePropertiesAndSchema.nodeSchema(), - relationships.relationshipSchema(relationshipType), + relationshipSchema, Map.of() ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java index a97f445a013..6fb4d930b93 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java @@ -415,10 +415,7 @@ public void addRelationshipType( ) { updateGraphStore(graphStore -> { graphStore.relationships.computeIfAbsent(relationshipType, __ -> { - var relationshipSchemaEntry = schema - .relationshipSchema() - .getOrCreateRelationshipType(relationshipType, relationships.direction()); - relationships.updateRelationshipSchemaEntry(relationshipSchemaEntry); + schema().relationshipSchema().set(relationships.relationshipSchemaEntry()); return relationships; }); }); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index e52909d7275..33d018d5de7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -307,7 +307,8 @@ public static HugeGraph create(IdMap idMap, SingleTypeRelationships relationship assert relationshipPropertyStore.values().size() == 1: "Cannot instantiate graph with more than one relationship property."; }); - var relationshipSchema = relationships.relationshipSchema(RelationshipType.of("REL")); + RelationshipSchema relationshipSchema = RelationshipSchema.empty(); + relationshipSchema.set(relationships.relationshipSchemaEntry()); return create( MutableGraphSchema.of(nodeSchema, relationshipSchema, Map.of()), diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index fdf82225a33..bb6ae7c1274 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -29,6 +29,8 @@ import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; +import org.neo4j.gds.api.schema.ImmutableGraphSchema; +import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.compat.CompatUserAggregator; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.ConfigKeyValidation; @@ -58,7 +60,6 @@ import org.neo4j.values.virtual.MapValue; import org.neo4j.values.virtual.MapValueBuilder; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -532,20 +533,17 @@ private void buildRelationshipsWithProperties( AdjacencyCompressor.ValueMapper valueMapper ) { var relationshipImportResultBuilder = RelationshipImportResult.builder(); - var relationshipSchemas = new HashMap(); + var relationshipSchema = RelationshipSchema.empty(); this.relImporters.forEach((relationshipType, relImporter) -> { var relationships = relImporter.build( Optional.of(valueMapper), Optional.empty() ); - var schema = relationships.relationshipSchema(relationshipType); - relationshipSchemas.put(relationshipType, schema.get(relationshipType)); + relationshipSchema.set(relationships.relationshipSchemaEntry()); relationshipImportResultBuilder.putImportResult(relationshipType, relationships); }); - var relationshipSchema = new MutableRelationshipSchema(relationshipSchemas); - graphStoreBuilder.relationshipImportResult(relationshipImportResultBuilder.build()); this.graphSchemaBuilder.relationshipSchema(relationshipSchema); diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index 69f751286e8..b9dda03539c 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -162,12 +162,11 @@ protected ProgressTracker initProgressTracker() { @Override protected MutableGraphSchema computeGraphSchema(Nodes nodes, RelationshipImportResult relationshipImportResult) { - var relationshipSchema = relationshipImportResult.importResults().entrySet().stream().reduce( + var relationshipSchema = relationshipImportResult.importResults().values().stream().reduce( MutableRelationshipSchema.empty(), - (unionSchema, entry) -> { - var relationshipType = entry.getKey(); - var relationships = entry.getValue(); - return unionSchema.union(relationships.relationshipSchema(relationshipType)); + (unionSchema, relationships) -> { + unionSchema.set(relationships.relationshipSchemaEntry()); + return unionSchema; }, MutableRelationshipSchema::union ); From c6a4751513732061d5c5763dffb41b0f0c171a54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 14:25:02 +0100 Subject: [PATCH 156/400] Make sure all usages of RelationshipsBuilder initialize it with a relationship type --- .../indexInverse/InverseRelationships.java | 5 +- .../gds/beta/undirected/ToUndirected.java | 11 ++++- .../neo4j/gds/beta/walking/CollapsePath.java | 5 ++ .../walking/CollapsePathAlgorithmFactory.java | 1 + .../gds/leiden/GraphAggregationPhase.java | 1 + .../java/org/neo4j/gds/louvain/Louvain.java | 2 + .../similarity/SimilarityGraphBuilder.java | 2 + .../beta/walking/CollapseMultiPathsTest.java | 2 + .../gds/beta/walking/CollapsePathTest.java | 3 ++ .../gds/embeddings/fastrp/FastRPTest.java | 4 +- .../gds/embeddings/hashgnn/HashGNNTest.java | 5 +- .../gds/embeddings/node2vec/Node2VecTest.java | 5 +- .../beta/generator/RandomGraphGenerator.java | 1 + .../gds/core/loading/CSRGraphStoreUtil.java | 1 + .../loading/CypherRelationshipLoader.java | 1 + .../loading/RelationshipImportResult.java | 39 +++++++++++---- .../core/loading/SingleTypeRelationships.java | 1 + .../loading/construction/GraphFactory.java | 4 +- .../SingleTypeRelationshipsBuilder.java | 47 ++++++++++++++++++- .../gds/core/huge/TransientCsrListTest.java | 4 +- .../gds/core/loading/CSRGraphStoreTest.java | 3 ++ .../gds/core/loading/GraphStoreTest.java | 4 +- .../neo4j/gds/projection/GraphAggregator.java | 1 + .../io/GraphStoreRelationshipVisitor.java | 9 ++-- .../io/file/FileToGraphStoreImporter.java | 40 ++++------------ .../GraphStoreRelationshipVisitorTest.java | 23 ++++----- .../ml/splitting/DirectedEdgeSplitter.java | 5 +- .../neo4j/gds/ml/splitting/EdgeSplitter.java | 23 +++++++-- .../gds/ml/splitting/SplitRelationships.java | 4 ++ .../ml/splitting/UndirectedEdgeSplitter.java | 5 +- .../splitting/DirectedEdgeSplitterTest.java | 45 ++++++++++++++++-- .../splitting/RandomNegativeSamplerTest.java | 5 +- .../splitting/UndirectedEdgeSplitterTest.java | 27 +++++++++++ .../UserInputNegativeSamplerTest.java | 5 +- .../LinkPredictionRelationshipSampler.java | 32 +++++++++++-- ...hStreamRelationshipPropertiesProcTest.java | 9 +++- .../LinkPredictionPipelineMutateProc.java | 11 +++-- .../ShortestPathMutateResultConsumer.java | 1 + .../spanningtree/SpanningTreeMutateSpec.java | 4 +- .../paths/steiner/SteinerTreeMutateSpec.java | 4 +- .../TraverseMutateResultConsumer.java | 1 + .../filteredknn/FilteredKnnMutateProc.java | 4 +- .../FilteredNodeSimilarityMutateSpec.java | 8 +++- .../gds/similarity/knn/KnnMutateProc.java | 1 + .../nodesim/NodeSimilarityMutateProc.java | 9 +++- .../gds/beta/filter/RelationshipsFilter.java | 1 + .../java/org/neo4j/gds/gdl/GdlFactory.java | 1 + 47 files changed, 335 insertions(+), 94 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/beta/indexInverse/InverseRelationships.java b/algo/src/main/java/org/neo4j/gds/beta/indexInverse/InverseRelationships.java index 3062b2f9c24..78a2bf16940 100644 --- a/algo/src/main/java/org/neo4j/gds/beta/indexInverse/InverseRelationships.java +++ b/algo/src/main/java/org/neo4j/gds/beta/indexInverse/InverseRelationships.java @@ -79,7 +79,7 @@ public Map compute() { .propertySchemasFor(fromRelationshipType); var propertyKeys = propertySchemas.stream().map(PropertySchema::key).collect(Collectors.toList()); - var relationshipsBuilder = initializeRelationshipsBuilder(propertySchemas); + var relationshipsBuilder = initializeRelationshipsBuilder(fromRelationshipType, propertySchemas); var tasks = createTasks(fromRelationshipType, propertyKeys, relationshipsBuilder); @@ -109,8 +109,9 @@ public Map compute() { } @NotNull - private RelationshipsBuilder initializeRelationshipsBuilder(List propertySchemas) { + private RelationshipsBuilder initializeRelationshipsBuilder(RelationshipType relationshipType, List propertySchemas) { RelationshipsBuilderBuilder relationshipsBuilderBuilder = GraphFactory.initRelationshipsBuilder() + .relationshipType(relationshipType) .concurrency(config.concurrency()) .nodes(graphStore.nodes()) .executorService(executorService) diff --git a/algo/src/main/java/org/neo4j/gds/beta/undirected/ToUndirected.java b/algo/src/main/java/org/neo4j/gds/beta/undirected/ToUndirected.java index 6eb1bdc7323..9d102b3d606 100644 --- a/algo/src/main/java/org/neo4j/gds/beta/undirected/ToUndirected.java +++ b/algo/src/main/java/org/neo4j/gds/beta/undirected/ToUndirected.java @@ -74,7 +74,10 @@ public SingleTypeRelationships compute() { .propertySchemasFor(fromRelationshipType); var propertyKeys = propertySchemas.stream().map(PropertySchema::key).collect(Collectors.toList()); - var relationshipsBuilder = initializeRelationshipsBuilder(propertySchemas); + var relationshipsBuilder = initializeRelationshipsBuilder( + RelationshipType.of(config.mutateRelationshipType()), + propertySchemas + ); var tasks = createTasks(fromRelationshipType, propertyKeys, relationshipsBuilder); @@ -101,8 +104,12 @@ public SingleTypeRelationships compute() { } @NotNull - private RelationshipsBuilder initializeRelationshipsBuilder(List propertySchemas) { + private RelationshipsBuilder initializeRelationshipsBuilder( + RelationshipType relationshipType, + List propertySchemas + ) { RelationshipsBuilderBuilder relationshipsBuilderBuilder = GraphFactory.initRelationshipsBuilder() + .relationshipType(relationshipType) .concurrency(config.concurrency()) .nodes(graphStore.nodes()) .executorService(executorService) diff --git a/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePath.java b/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePath.java index a2dfe2cecd1..2378f1cbd48 100644 --- a/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePath.java +++ b/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePath.java @@ -21,6 +21,7 @@ import org.neo4j.gds.Algorithm; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.concurrency.ParallelUtil; @@ -43,12 +44,14 @@ public class CollapsePath extends Algorithm { private final List pathTemplates; private final long nodeCount; private final boolean allowSelfLoops; + private RelationshipType mutateRelationshipType; private final int concurrency; private final ExecutorService executorService; public CollapsePath( List pathTemplates, boolean allowSelfLoops, + RelationshipType mutateRelationshipType, int concurrency, ExecutorService executorService ) { @@ -56,6 +59,7 @@ public CollapsePath( this.pathTemplates = pathTemplates; this.nodeCount = pathTemplates.get(0)[0].nodeCount(); this.allowSelfLoops = allowSelfLoops; + this.mutateRelationshipType = mutateRelationshipType; this.concurrency = concurrency; this.executorService = executorService; } @@ -64,6 +68,7 @@ public CollapsePath( public SingleTypeRelationships compute() { RelationshipsBuilder relImporter = GraphFactory.initRelationshipsBuilder() .nodes(pathTemplates.get(0)[0]) // just need any arbitrary graph + .relationshipType(mutateRelationshipType) .orientation(Orientation.NATURAL) .aggregation(Aggregation.NONE) .concurrency(concurrency) diff --git a/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePathAlgorithmFactory.java b/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePathAlgorithmFactory.java index 1a69af28c0c..9321d6cd310 100644 --- a/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePathAlgorithmFactory.java +++ b/algo/src/main/java/org/neo4j/gds/beta/walking/CollapsePathAlgorithmFactory.java @@ -64,6 +64,7 @@ public CollapsePath build( return new CollapsePath( pathTemplatesEncodedAsListsOfSingleRelationshipTypeGraphs, config.allowSelfLoops(), + RelationshipType.of(config.mutateRelationshipType()), config.concurrency(), Pools.DEFAULT ); diff --git a/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java b/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java index fbd2e4450ae..a9f65e589cf 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java @@ -148,6 +148,7 @@ Graph run() { IdMap idMap = nodesBuilder.build().idMap(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(RelationshipType.of("REL")) .orientation(direction.toOrientation()) .addPropertyConfig(GraphFactory.PropertyConfig.builder() .propertyKey("property") diff --git a/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java b/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java index 7bfbd9528c8..5c2694f2824 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java @@ -20,6 +20,7 @@ package org.neo4j.gds.louvain; import org.neo4j.gds.Algorithm; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.RelationshipIterator; @@ -233,6 +234,7 @@ private Graph summarizeGraph( IdMap idMap = nodesBuilder.build().idMap(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(RelationshipType.of("REL")) .orientation(rootGraph.schema().direction().toOrientation()) .addPropertyConfig(GraphFactory.PropertyConfig.builder() .propertyKey("property") diff --git a/algo/src/main/java/org/neo4j/gds/similarity/SimilarityGraphBuilder.java b/algo/src/main/java/org/neo4j/gds/similarity/SimilarityGraphBuilder.java index 9eddd7ab32c..241e0468d5b 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/SimilarityGraphBuilder.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/SimilarityGraphBuilder.java @@ -20,6 +20,7 @@ package org.neo4j.gds.similarity; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.core.compress.AdjacencyListBehavior; @@ -84,6 +85,7 @@ public SimilarityGraphBuilder( public Graph build(Stream stream) { var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap.rootIdMap()) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of("property")) .concurrency(concurrency) diff --git a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapseMultiPathsTest.java b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapseMultiPathsTest.java index fea823762b4..ae8d4f754ef 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapseMultiPathsTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapseMultiPathsTest.java @@ -108,6 +108,7 @@ void shouldFollowRoutesFromMovies() { var path = new CollapsePath( pathTemplates, false, + RelationshipType.of("REL"), 2, Pools.DEFAULT @@ -170,6 +171,7 @@ void shouldTurnEveryRouteIntoRelationship() { new Graph[]{plane} ), false, + RelationshipType.of("REL"), 2, Pools.DEFAULT diff --git a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java index 3e7b0da30cb..f97768619ab 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java @@ -119,6 +119,7 @@ void testCreatingRelationships() { var relationships = new CollapsePath( Collections.singletonList(new Graph[]{tookRel, tookRel}), false, + RelationshipType.of("SAME_DRUG"), 2, Pools.DEFAULT @@ -134,6 +135,7 @@ void testAllowCreatingSelfLoops() { var relationships = new CollapsePath( Collections.singletonList(new Graph[]{tookRel, tookRel}), true, + RelationshipType.of("SAME_DRUG"), 2, Pools.DEFAULT @@ -147,6 +149,7 @@ void runWithDifferentRelationshipTypes() { var relationships = new CollapsePath( Collections.singletonList(new Graph[]{tookGraph, takenByGraph}), false, + RelationshipType.of("SAME_DRUG"), 2, Pools.DEFAULT ).compute(); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java index ea8976c75e4..6c089f07b73 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/fastrp/FastRPTest.java @@ -42,13 +42,13 @@ import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.ml.core.features.FeatureExtraction; import org.neo4j.gds.ml.core.features.FeatureExtractor; -import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import java.util.List; import java.util.Optional; @@ -624,6 +624,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { ); RelationshipsBuilder firstRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(firstIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); @@ -647,6 +648,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { ); RelationshipsBuilder secondRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(secondIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java index 97686dd1295..083c6bd4b17 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.ResourceUtil; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; @@ -45,11 +46,11 @@ import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; -import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import java.util.List; import java.util.Map; @@ -372,6 +373,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { ); RelationshipsBuilder firstRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(firstIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); @@ -395,6 +397,7 @@ void shouldBeDeterministicGivenSameOriginalIds() { ); RelationshipsBuilder secondRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(secondIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java index 6e98729e9fb..06712e2221c 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/node2vec/Node2VecTest.java @@ -36,6 +36,7 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.StoreLoaderBuilder; import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.api.Graph; @@ -53,10 +54,10 @@ import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import org.neo4j.gds.embeddings.node2vec.Node2VecBaseConfig.EmbeddingInitializer; import org.neo4j.gds.gdl.GdlFactory; import org.neo4j.gds.ml.core.tensor.FloatVector; -import org.neo4j.gds.core.utils.shuffle.ShuffleUtil; import java.util.List; import java.util.Optional; @@ -277,6 +278,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit ); RelationshipsBuilder firstRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(firstIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); @@ -298,6 +300,7 @@ void shouldBeFairlyConsistentUnderOriginalIds(EmbeddingInitializer embeddingInit ); RelationshipsBuilder secondRelationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(secondIdMap) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .executorService(Pools.DEFAULT) .build(); diff --git a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java index dc4f640ff9b..e0e01eae749 100644 --- a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java +++ b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java @@ -128,6 +128,7 @@ public HugeGraph generate() { var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(relationshipType) .orientation(direction.toOrientation()) .addAllPropertyConfigs(maybeRelationshipPropertyProducer .map(propertyProducer -> List.of(GraphFactory.PropertyConfig.of( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index e4f348110ac..c356e5d1a7a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -88,6 +88,7 @@ public static CSRGraphStore createFromGraph( var relationshipImportResult = RelationshipImportResult.builder().putImportResult( relationshipType, SingleTypeRelationships.builder() + .relationshipSchemaEntry(entry) .topology(graph.relationshipTopology()) .properties(relationshipProperties) .direction(direction) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java index 8e9479fe40a..f372a453a15 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java @@ -179,6 +179,7 @@ RelationshipsBuilder getOrCreateRelationshipsBuilder(RelationshipType relationsh private RelationshipsBuilder createRelationshipsBuilder(RelationshipType relationshipType) { return GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(relationshipType) .concurrency(cypherConfig.readConcurrency()) .propertyConfigs(propertyConfigs) .orientation(Orientation.NATURAL) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java index 354f95b9ff3..72760220ba3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java @@ -32,6 +32,7 @@ import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.ValueTypes; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import org.neo4j.values.storable.NumberType; import java.util.Collection; @@ -56,14 +57,20 @@ static RelationshipImportResult of( ) { var relationshipImportResultBuilder = RelationshipImportResult.builder(); - topologies.forEach((relationshipType, topology) -> relationshipImportResultBuilder.putImportResult( - relationshipType, - SingleTypeRelationships.builder() - .topology(topology) - .properties(Optional.ofNullable(properties.get(relationshipType))) - .direction(directions.get(relationshipType)) - .build() - )); + topologies.forEach((relationshipType, topology) -> { + Direction direction = directions.get(relationshipType); + var schemaEntry = new RelationshipSchemaEntry(relationshipType, direction); + + + relationshipImportResultBuilder.putImportResult( + relationshipType, + SingleTypeRelationships.builder() + .topology(topology) + .properties(Optional.ofNullable(properties.get(relationshipType))) + .direction(direction) + .build() + ); + }); return relationshipImportResultBuilder.build(); } @@ -105,9 +112,23 @@ static RelationshipImportResult of(Collection{ + props.relationshipProperties().forEach((key, prop) -> { + schemaEntry.addProperty(key, prop.valueType(), prop.propertyState()); + }); + }); + var importResultBuilder = builders.computeIfAbsent( importContext.relationshipType(), - relationshipType -> SingleTypeRelationships.builder().direction(direction) + relationshipType -> SingleTypeRelationships + .builder() + .direction(direction) + .relationshipSchemaEntry(schemaEntry) ); if (isInverseRelationship) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java index df01f1ea263..b3bb0b74319 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java @@ -39,6 +39,7 @@ public interface SingleTypeRelationships { SingleTypeRelationships EMPTY = SingleTypeRelationships .builder() + .relationshipSchemaEntry(new RelationshipSchemaEntry(RelationshipType.of("REL"), Direction.DIRECTED)) .direction(Direction.DIRECTED) .topology(Topology.EMPTY) .build(); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 33d018d5de7..161a0aa5a56 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -37,6 +37,7 @@ import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; import org.neo4j.gds.api.schema.NodeSchema; +import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; import org.neo4j.gds.core.concurrency.Pools; @@ -187,6 +188,7 @@ public static RelationshipsBuilderBuilder initRelationshipsBuilder() { @Builder.Factory static RelationshipsBuilder relationshipsBuilder( PartialIdMap nodes, + RelationshipType relationshipType, Optional orientation, List propertyConfigs, Optional aggregation, @@ -204,7 +206,6 @@ static RelationshipsBuilder relationshipsBuilder( .map(Aggregation::resolve) .toArray(Aggregation[]::new); - var relationshipType = RelationshipType.ALL_RELATIONSHIPS; var isMultiGraph = Arrays.stream(aggregations).allMatch(Aggregation::equivalentToNone); var actualOrientation = orientation.orElse(Orientation.NATURAL); @@ -259,6 +260,7 @@ static RelationshipsBuilder relationshipsBuilder( .idMap(nodes) .importer(singleTypeRelationshipImporter) .bufferSize(bufferSize) + .relationshipType(relationshipType) .propertyConfigs(propertyConfigs) .isMultiGraph(isMultiGraph) .loadRelationshipProperty(loadRelationshipProperties) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java index 037203c9089..3d9d73a89f9 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java @@ -20,6 +20,7 @@ package org.neo4j.gds.core.loading.construction; import org.immutables.builder.Builder; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.ImmutableProperties; import org.neo4j.gds.api.ImmutableRelationshipProperty; @@ -29,6 +30,8 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.ImmutableRelationshipPropertySchema; +import org.neo4j.gds.api.schema.RelationshipPropertySchema; +import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import org.neo4j.gds.core.compress.AdjacencyCompressor; import org.neo4j.gds.core.compress.AdjacencyListsWithProperties; import org.neo4j.gds.core.concurrency.RunWithConcurrency; @@ -47,6 +50,7 @@ abstract class SingleTypeRelationshipsBuilder { final PartialIdMap idMap; final int bufferSize; + final RelationshipType relationshipType; final List propertyConfigs; final boolean isMultiGraph; @@ -62,6 +66,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( SingleTypeRelationshipImporter importer, Optional inverseImporter, int bufferSize, + RelationshipType relationshipType, List propertyConfigs, boolean isMultiGraph, boolean loadRelationshipProperty, @@ -75,6 +80,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( importer, inverseImporter.get(), bufferSize, + relationshipType, propertyConfigs, isMultiGraph, loadRelationshipProperty, @@ -86,6 +92,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( idMap, importer, bufferSize, + relationshipType, propertyConfigs, isMultiGraph, loadRelationshipProperty, @@ -98,6 +105,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( SingleTypeRelationshipsBuilder( PartialIdMap idMap, int bufferSize, + RelationshipType relationshipType, List propertyConfigs, boolean isMultiGraph, boolean loadRelationshipProperty, @@ -107,6 +115,7 @@ static SingleTypeRelationshipsBuilder singleTypeRelationshipsBuilder( ) { this.idMap = idMap; this.bufferSize = bufferSize; + this.relationshipType = relationshipType; this.propertyConfigs = propertyConfigs; this.isMultiGraph = isMultiGraph; this.loadRelationshipProperty = loadRelationshipProperty; @@ -143,6 +152,28 @@ SingleTypeRelationships build( return singleTypeRelationshipImportResult(); } + RelationshipSchemaEntry relationshipSchemaEntry(Optional properties) { + RelationshipSchemaEntry entry = new RelationshipSchemaEntry( + relationshipType, + direction + ); + + properties.ifPresent(relationshipPropertyStore -> relationshipPropertyStore + .relationshipProperties() + .forEach((propertyKey, relationshipProperty) -> entry.addProperty( + propertyKey, + RelationshipPropertySchema.of(propertyKey, + relationshipProperty.valueType(), + relationshipProperty.defaultValue(), + relationshipProperty.propertyState(), + relationshipProperty.aggregation() + ) + )) + ); + + return entry; + } + RelationshipPropertyStore relationshipPropertyStore(AdjacencyListsWithProperties adjacencyListsWithProperties) { var propertyStoreBuilder = RelationshipPropertyStore.builder(); @@ -186,6 +217,7 @@ static class NonIndexed extends SingleTypeRelationshipsBuilder { PartialIdMap idMap, SingleTypeRelationshipImporter importer, int bufferSize, + RelationshipType relationshipType, List propertyConfigs, boolean isMultiGraph, boolean loadRelationshipProperty, @@ -196,6 +228,7 @@ static class NonIndexed extends SingleTypeRelationshipsBuilder { super( idMap, bufferSize, + relationshipType, propertyConfigs, isMultiGraph, loadRelationshipProperty, @@ -235,11 +268,15 @@ SingleTypeRelationships singleTypeRelationshipImportResult() { .topology(topology) .direction(this.direction); + RelationshipPropertyStore properties = null; if (loadRelationshipProperty) { - var properties = relationshipPropertyStore(adjacencyListsWithProperties); + properties = relationshipPropertyStore(adjacencyListsWithProperties); singleRelationshipTypeImportResultBuilder.properties(properties); } + singleRelationshipTypeImportResultBuilder + .relationshipSchemaEntry(relationshipSchemaEntry(Optional.ofNullable(properties))); + return singleRelationshipTypeImportResultBuilder.build(); } } @@ -254,6 +291,7 @@ static class Indexed extends SingleTypeRelationshipsBuilder { SingleTypeRelationshipImporter forwardImporter, SingleTypeRelationshipImporter inverseImporter, int bufferSize, + RelationshipType relationshipType, List propertyConfigs, boolean isMultiGraph, boolean loadRelationshipProperty, @@ -264,6 +302,7 @@ static class Indexed extends SingleTypeRelationshipsBuilder { super( idMap, bufferSize, + relationshipType, propertyConfigs, isMultiGraph, loadRelationshipProperty, @@ -319,12 +358,16 @@ SingleTypeRelationships singleTypeRelationshipImportResult() { .inverseTopology(inverseTopology) .direction(this.direction); + RelationshipPropertyStore forwardProperties = null; if (loadRelationshipProperty) { - var forwardProperties = relationshipPropertyStore(forwardListWithProperties); + forwardProperties = relationshipPropertyStore(forwardListWithProperties); var inverseProperties = relationshipPropertyStore(inverseListWithProperties); singleRelationshipTypeImportResultBuilder.properties(forwardProperties).inverseProperties(inverseProperties); } + singleRelationshipTypeImportResultBuilder + .relationshipSchemaEntry(relationshipSchemaEntry(Optional.ofNullable(forwardProperties))); + return singleRelationshipTypeImportResultBuilder.build(); } } diff --git a/core/src/test/java/org/neo4j/gds/core/huge/TransientCsrListTest.java b/core/src/test/java/org/neo4j/gds/core/huge/TransientCsrListTest.java index 2a742ec472b..4d89854a8f7 100644 --- a/core/src/test/java/org/neo4j/gds/core/huge/TransientCsrListTest.java +++ b/core/src/test/java/org/neo4j/gds/core/huge/TransientCsrListTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.AdjacencyCursor; import org.neo4j.gds.api.IdMap; @@ -202,6 +203,7 @@ void shouldWorkWithVeryDenseNodes(TestMethodRunner runner, long firstDegree, lon var relsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(nodes.idMap()) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .build(); @@ -254,7 +256,6 @@ void shouldWorkWithVeryDenseNodes(TestMethodRunner runner, long firstDegree, lon }); } - static AdjacencyCursor adjacencyCursorFromTargets(long[] targets) { long sourceNodeId = targets[0]; NodesBuilder nodesBuilder = GraphFactory.initNodesBuilder() @@ -268,6 +269,7 @@ static AdjacencyCursor adjacencyCursorFromTargets(long[] targets) { RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(RelationshipType.of("REL")) .concurrency(1) .executorService(Pools.DEFAULT) .build(); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java index 94e088a1bc1..a0ad1b2c2cc 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java @@ -68,6 +68,7 @@ void addRelationshipTypeUndirected() { RelationshipsBuilder relBuilder = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(RelationshipType.of("REL")) .orientation(Orientation.UNDIRECTED) .build(); @@ -95,6 +96,7 @@ void addRelationshipTypeDirected() { RelationshipsBuilder relBuilder = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(RelationshipType.of("NEW")) .orientation(Orientation.NATURAL) .build(); @@ -166,6 +168,7 @@ void addRelationshipTypeMixed(Orientation baseOrientation, int baseRelCount, Ori RelationshipsBuilder relBuilder = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(RelationshipType.of("REL")) .orientation(addedOrientation) .build(); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java index eae8ef0a836..f70dfca558f 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java @@ -171,9 +171,11 @@ void testModificationDate() throws InterruptedException { // add relationships Thread.sleep(42); + RelationshipType bar = RelationshipType.of("BAR"); graphStore.addRelationshipType( - RelationshipType.of("BAR"), + bar, SingleTypeRelationships.of( + bar, ImmutableTopology.of(CompressedAdjacencyList.EMPTY, 0, false), Direction.DIRECTED, Optional.empty(), diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index bb6ae7c1274..a7160ee3bb4 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -467,6 +467,7 @@ private RelationshipsBuilder newRelImporter(RelationshipType relType, @Nullable var relationshipsBuilderBuilder = GraphFactory.initRelationshipsBuilder() .nodes(this.idMapBuilder) + .relationshipType(relType) .orientation(orientation) .aggregation(Aggregation.NONE) .indexInverse(indexInverse) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreRelationshipVisitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreRelationshipVisitor.java index 6dfab81155b..dae414f417c 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreRelationshipVisitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreRelationshipVisitor.java @@ -62,16 +62,19 @@ protected void exportElement() { // TODO: this logic should move to the RelationshipsBuilder var relationshipsBuilder = relationshipFromVisitorBuilders.computeIfAbsent( relationshipType(), - (relationshipType) -> { + (relationshipTypeString) -> { var propertyConfigs = getPropertySchema() .stream() .map(schema -> GraphFactory.PropertyConfig.of(schema.key(), schema.aggregation(), schema.defaultValue())) .collect(Collectors.toList()); + RelationshipType relationshipType = RelationshipType.of(relationshipTypeString); + var relBuilder = relationshipBuilderSupplier.get() + .relationshipType(relationshipType) .propertyConfigs(propertyConfigs) - .indexInverse(inverseIndexedRelationshipTypes.contains(RelationshipType.of(relationshipType))) + .indexInverse(inverseIndexedRelationshipTypes.contains(relationshipType)) .build(); - relationshipBuilders.put(relationshipType, relBuilder); + relationshipBuilders.put(relationshipTypeString, relBuilder); return RelationshipBuilderFromVisitor.of( propertyConfigs.size(), relBuilder, diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 9ca4493139d..ba3f0d48798 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -36,6 +36,7 @@ import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; +import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodesBuilder; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; @@ -51,7 +52,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -194,12 +194,7 @@ private void importRelationships(FileInput fileInput, IdMap nodes) { ParallelUtil.run(tasks, Pools.DEFAULT); - var relationships = relationshipTopologyAndProperties(relationshipBuildersByType); - var relationshipImportResult = RelationshipImportResult.of( - relationships.topologies(), - relationships.properties(), - relationshipSchema.directions() - ); + var relationshipImportResult = relationshipImportResult(relationshipBuildersByType); graphStoreBuilder.relationshipImportResult(relationshipImportResult); @@ -238,30 +233,15 @@ private void importGraphProperties(FileInput fileInput) { } } - public static RelationshipTopologyAndProperties relationshipTopologyAndProperties(Map relationshipBuildersByType) { - var propertyStores = new HashMap(); - var relationshipTypeTopologyMap = relationshipTypeToTopologyMapping( - relationshipBuildersByType, - propertyStores - ); - - var importedRelationships = relationshipTypeTopologyMap.values().stream().mapToLong(Topology::elementCount).sum(); - return ImmutableRelationshipTopologyAndProperties.of(relationshipTypeTopologyMap, propertyStores, importedRelationships); - } + public static RelationshipImportResult relationshipImportResult(Map relationshipBuildersByType) { + Map foo = relationshipBuildersByType.entrySet() + .stream() + .collect(Collectors.toMap( + e -> RelationshipType.of(e.getKey()), + e -> e.getValue().build() + )); - private static Map relationshipTypeToTopologyMapping( - Map relationshipBuildersByType, - Map propertyStores - ) { - return relationshipBuildersByType.entrySet().stream().map(entry -> { - var relationshipType = RelationshipType.of(entry.getKey()); - var relationships = entry.getValue().build(); - - if (relationships.properties().isPresent()) { - propertyStores.put(relationshipType, relationships.properties().get()); - } - return Map.entry(relationshipType, relationships.topology()); - }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + return RelationshipImportResult.builder().importResults(foo).build(); } @ValueClass diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java index 96efedf45b3..e1d246264c1 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/file/GraphStoreRelationshipVisitorTest.java @@ -31,7 +31,6 @@ import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.ImmutableNodes; import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; -import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; import org.neo4j.gds.core.loading.construction.RelationshipsBuilderBuilder; @@ -150,24 +149,26 @@ private Graph createGraph( Map relationshipBuildersByType, long expectedImportedRelationshipsCount ) { - var actualRelationships = FileToGraphStoreImporter.relationshipTopologyAndProperties(relationshipBuildersByType); - assertThat(actualRelationships.importedRelationships()).isEqualTo(expectedImportedRelationshipsCount); + var actualRelationships = FileToGraphStoreImporter.relationshipImportResult(relationshipBuildersByType); + var actualRelationshipCount = actualRelationships + .importResults() + .values() + .stream() + .mapToLong(r -> r.topology().elementCount()) + .sum(); + + assertThat(actualRelationshipCount).isEqualTo(expectedImportedRelationshipsCount); var nodes = ImmutableNodes.builder() .idMap(expectedGraph) - .schema(MutableNodeSchema.from(expectedGraph.schema().nodeSchema())) + .schema(expectedGraph.schema().nodeSchema()) .build(); - Map propertyStores = actualRelationships.properties(); return new GraphStoreBuilder() - .schema(MutableGraphSchema.from(expectedGraph.schema())) + .schema(expectedGraph.schema()) .capabilities(ImmutableStaticCapabilities.of(true)) .nodes(nodes) - .relationshipImportResult(RelationshipImportResult.of( - actualRelationships.topologies(), - propertyStores, - expectedGraph.schema().relationshipSchema().directions() - )) + .relationshipImportResult(actualRelationships) .databaseId(DatabaseId.random()) .concurrency(1) .build() diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java index e7c00c7ab63..44b46d421ec 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.predicates.LongLongPredicate; import org.apache.commons.lang3.mutable.MutableLong; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; @@ -37,9 +38,11 @@ public DirectedEdgeSplitter( Optional maybeSeed, IdMap sourceLabels, IdMap targetLabels, + RelationshipType selectedRelationshipType, + RelationshipType remainingRelationshipType, int concurrency ) { - super(maybeSeed, sourceLabels, targetLabels, concurrency); + super(maybeSeed, sourceLabels, targetLabels, selectedRelationshipType, remainingRelationshipType, concurrency); } @Override diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java index bb45719edd1..8e6d34ece1e 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java @@ -22,6 +22,7 @@ import com.carrotsearch.hppc.predicates.LongLongPredicate; import com.carrotsearch.hppc.predicates.LongPredicate; import org.apache.commons.lang3.mutable.MutableLong; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.Graph; @@ -41,12 +42,24 @@ public abstract class EdgeSplitter { public static final double POSITIVE = 1D; public static final String RELATIONSHIP_PROPERTY = "label"; private final Random rng; + private final RelationshipType selectedRelationshipType; + private final RelationshipType remainingRelationshipType; protected final IdMap sourceNodes; protected final IdMap targetNodes; + protected int concurrency; - EdgeSplitter(Optional maybeSeed, IdMap sourceNodes, IdMap targetNodes, int concurrency) { + EdgeSplitter( + Optional maybeSeed, + IdMap sourceNodes, + IdMap targetNodes, + RelationshipType selectedRelationshipType, + RelationshipType remainingRelationshipType, + int concurrency + ) { + this.selectedRelationshipType = selectedRelationshipType; + this.remainingRelationshipType = remainingRelationshipType; this.rng = new Random(); maybeSeed.ifPresent(rng::setSeed); @@ -66,7 +79,9 @@ public SplitResult splitPositiveExamples( RelationshipsBuilder selectedRelsBuilder = newRelationshipsBuilder( graph, - Direction.DIRECTED, Optional.of(EdgeSplitter.RELATIONSHIP_PROPERTY) + selectedRelationshipType, + Direction.DIRECTED, + Optional.of(EdgeSplitter.RELATIONSHIP_PROPERTY) ); Direction remainingRelDirection = graph.schema().direction(); @@ -74,7 +89,7 @@ public SplitResult splitPositiveExamples( RelationshipsBuilder remainingRelsBuilder; RelationshipWithPropertyConsumer remainingRelsConsumer; - remainingRelsBuilder = newRelationshipsBuilder(graph, remainingRelDirection, remainingRelPropertyKey); + remainingRelsBuilder = newRelationshipsBuilder(graph, remainingRelationshipType, remainingRelDirection, remainingRelPropertyKey); remainingRelsConsumer = (s, t, w) -> { remainingRelsBuilder.addFromInternal(graph.toRootNodeId(s), graph.toRootNodeId(t), w); return true; @@ -139,10 +154,12 @@ protected long samplesPerNode(long maxSamples, double remainingSamples, long rem private static RelationshipsBuilder newRelationshipsBuilder( Graph graph, + RelationshipType relationshipType, Direction direction, Optional propertyKey ) { return GraphFactory.initRelationshipsBuilder() + .relationshipType(relationshipType) .aggregation(Aggregation.SINGLE) .nodes(graph) .orientation(direction.toOrientation()) diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java index f67d9c8ff02..f09e8e3e5df 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java @@ -100,12 +100,16 @@ public EdgeSplitter.SplitResult compute() { config.randomSeed(), sourceNodes, targetNodes, + config.holdoutRelationshipType(), + config.remainingRelationshipType(), config.concurrency() ) : new DirectedEdgeSplitter( config.randomSeed(), sourceNodes, targetNodes, + config.holdoutRelationshipType(), + config.remainingRelationshipType(), config.concurrency() ); diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitter.java index 778d501d886..753db215cde 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitter.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.predicates.LongLongPredicate; import org.apache.commons.lang3.mutable.MutableLong; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; @@ -46,9 +47,11 @@ public UndirectedEdgeSplitter( Optional maybeSeed, IdMap sourceNodes, IdMap targetNodes, + RelationshipType selectedRelationshipType, + RelationshipType remainingRelationshipType, int concurrency ) { - super(maybeSeed, sourceNodes, targetNodes, concurrency); + super(maybeSeed, sourceNodes, targetNodes, selectedRelationshipType, remainingRelationshipType, concurrency); } @Override diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java index 8e06563dbeb..7b11bf164cb 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.PropertyCursor; import org.neo4j.gds.api.schema.Direction; @@ -130,6 +131,8 @@ void splitSkewedGraph() { Optional.of(-1L), skewedGraphStore.nodes(), skewedGraphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -147,6 +150,8 @@ void splitMultiGraph() { Optional.of(-1L), multiGraphStore.nodes(), multiGraphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -165,6 +170,8 @@ void split() { Optional.of(-1L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -199,7 +206,13 @@ void negativeEdgesShouldNotOverlapMasterGraph() { .build() .generate(); - var splitter = new DirectedEdgeSplitter(Optional.of(42L), huuuuugeDenseGraph, huuuuugeDenseGraph, 4); + var splitter = new DirectedEdgeSplitter(Optional.of(42L), + huuuuugeDenseGraph, + huuuuugeDenseGraph, + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), + 4 + ); var splitResult = splitter.splitPositiveExamples(huuuuugeDenseGraph, 0.9, Optional.empty()); var graph = GraphFactory.create( huuuuugeDenseGraph.idMap(), @@ -224,7 +237,14 @@ void negativeEdgesShouldNotOverlapMasterGraph() { @Test void negativeEdgeSampling() { - var splitter = new DirectedEdgeSplitter(Optional.of(42L), graphStore.nodes(), graphStore.nodes(), 4); + var splitter = new DirectedEdgeSplitter( + Optional.of(42L), + graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), + 4 + ); var sum = 0; for (int i = 0; i < 100; i++) { @@ -243,6 +263,8 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { Optional.of(1337L), multiLabelGraphStore.getGraph(sourceNodeLabels), multiLabelGraphStore.getGraph(targetNodeLabels), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -269,7 +291,14 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { @Test void samplesWithinBounds() { - var splitter = new DirectedEdgeSplitter(Optional.of(42L), graphStore.nodes(), graphStore.nodes(), 4); + var splitter = new DirectedEdgeSplitter( + Optional.of(42L), + graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), + 4 + ); assertEquals(1, splitter.samplesPerNode(1, 100, 10)); assertEquals(1, splitter.samplesPerNode(100, 1, 1)); @@ -277,7 +306,15 @@ void samplesWithinBounds() { @Test void shouldPreserveRelationshipWeights() { - var splitter = new DirectedEdgeSplitter(Optional.of(42L), graphStore.nodes(), graphStore.nodes(), 4); + var splitter = new DirectedEdgeSplitter( + Optional.of(42L), + graphStore.nodes(), + graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), + 4 + ); + EdgeSplitter.SplitResult split = splitter.splitPositiveExamples(graph, 0.01, Optional.of("foo")); var maybeProp = split.remainingRels().build().properties(); assertThat(maybeProp).isPresent(); diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/RandomNegativeSamplerTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/RandomNegativeSamplerTest.java index edc9afef118..bd86e9b287f 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/RandomNegativeSamplerTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/RandomNegativeSamplerTest.java @@ -20,6 +20,7 @@ package org.neo4j.gds.ml.splitting; import org.junit.jupiter.api.Test; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.Aggregation; @@ -72,10 +73,10 @@ void generateNegativeSamples() { RelationshipsBuilder testBuilder = new RelationshipsBuilderBuilder().nodes(graph).addPropertyConfig( GraphFactory.PropertyConfig.of("property", Aggregation.SINGLE, DefaultValue.forDouble()) - ).build(); + ).relationshipType(RelationshipType.of("TEST")).build(); RelationshipsBuilder trainBuilder = new RelationshipsBuilderBuilder().nodes(graph).addPropertyConfig( GraphFactory.PropertyConfig.of("property", Aggregation.SINGLE, DefaultValue.forDouble()) - ).build(); + ).relationshipType(RelationshipType.of("TRAIN")).build(); sampler.produceNegativeSamples(testBuilder, trainBuilder); diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java index c290c0397bb..ea044902700 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PropertyCursor; @@ -96,6 +97,8 @@ void split() { Optional.of(1337L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -124,6 +127,8 @@ void splitMultiGraph() { Optional.of(-1L), multiGraphStore.nodes(), multiGraphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -152,6 +157,8 @@ void negativeEdgesShouldNotOverlapMasterGraph() { Optional.of(42L), huuuuugeDenseGraph, huuuuugeDenseGraph, + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); var splitResult = splitter.splitPositiveExamples(huuuuugeDenseGraph, 0.9, Optional.empty()); @@ -193,12 +200,16 @@ void shouldProduceDeterministicResult() { Optional.of(12L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var splitResult2 = new UndirectedEdgeSplitter( Optional.of(12L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var remainingAreEqual = relationshipsAreEqual( @@ -233,12 +244,16 @@ void shouldProduceNonDeterministicResult() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var splitResult2 = new UndirectedEdgeSplitter( Optional.of(117L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var remainingAreEqual = relationshipsAreEqual( @@ -262,6 +277,8 @@ void negativeEdgeSampling() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -282,6 +299,8 @@ void splitWithFilteringWithSameSourceTargetLabels() { Optional.of(1337L), graphStore.getGraph(NodeLabel.of("A")), graphStore.getGraph(NodeLabel.of("A")), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -315,6 +334,8 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { Optional.of(1337L), multiLabelGraphStore.getGraph(sourceNodeLabels), multiLabelGraphStore.getGraph(targetNodeLabels), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -346,6 +367,8 @@ void samplesWithinBounds() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); @@ -359,6 +382,8 @@ void shouldPreserveRelationshipWeights() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); EdgeSplitter.SplitResult split = splitter.splitPositiveExamples(graph, 0.01, Optional.of("foo")); @@ -386,6 +411,8 @@ void zeroNegativeSamples() { Optional.of(1337L), graphStore.nodes(), graphStore.nodes(), + RelationshipType.of("SELECTED"), + RelationshipType.of("REMAINING"), 4 ); diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java index bb824901872..41edc76800b 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.Orientation; +import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.Aggregation; @@ -73,10 +74,10 @@ void generateNegativeSamples() { RelationshipsBuilder testBuilder = new RelationshipsBuilderBuilder().nodes(graph).addPropertyConfig( GraphFactory.PropertyConfig.of("property", Aggregation.SINGLE, DefaultValue.forDouble()) - ).build(); + ).relationshipType(RelationshipType.of("TEST")).build(); RelationshipsBuilder trainBuilder = new RelationshipsBuilderBuilder().nodes(graph).addPropertyConfig( GraphFactory.PropertyConfig.of("property", Aggregation.SINGLE, DefaultValue.forDouble()) - ).build(); + ).relationshipType(RelationshipType.of("TRAIN")).build(); sampler.produceNegativeSamples(testBuilder, trainBuilder); diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java index 803113deff0..8f64f2f6644 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java @@ -110,7 +110,15 @@ public void splitAndSampleRelationships( // Relationship sets: test, train, feature-input, test-complement. The nodes are always the same. // 1. Split base graph into test, test-complement terminationFlag.assertRunning(); - var testSplitResult = split(sourceNodes, targetNodes, graph, relationshipWeightProperty, splitConfig.testComplementRelationshipType(), splitConfig.testFraction()); + var testSplitResult = split( + sourceNodes, + targetNodes, + graph, + relationshipWeightProperty, + splitConfig.testRelationshipType(), + splitConfig.testComplementRelationshipType(), + splitConfig.testFraction() + ); // 2. Split test-complement into (labeled) train and feature-input. var testComplementGraph = graphStore.getGraph( trainConfig.nodeLabelIdentifiers(graphStore), @@ -119,7 +127,15 @@ public void splitAndSampleRelationships( ); terminationFlag.assertRunning(); - var trainSplitResult = split(sourceNodes, targetNodes, testComplementGraph, relationshipWeightProperty, splitConfig.featureInputRelationshipType(), splitConfig.trainFraction()); + var trainSplitResult = split( + sourceNodes, + targetNodes, + testComplementGraph, + relationshipWeightProperty, + splitConfig.trainRelationshipType(), + splitConfig.featureInputRelationshipType(), + splitConfig.trainFraction() + ); // 3. add negative examples to test and train NegativeSampler negativeSampler = NegativeSampler.of( @@ -150,7 +166,15 @@ public void splitAndSampleRelationships( progressTracker.endSubTask("Split relationships"); } - private EdgeSplitter.SplitResult split(IdMap sourceNodes, IdMap targetNodes, Graph graph, Optional relationshipWeightProperty, RelationshipType remainingRelType, double selectedFraction) { + private EdgeSplitter.SplitResult split( + IdMap sourceNodes, + IdMap targetNodes, + Graph graph, + Optional relationshipWeightProperty, + RelationshipType selectedRelType, + RelationshipType remainingRelType, + double selectedFraction + ) { if (!graph.schema().isUndirected()) { throw new IllegalArgumentException("EdgeSplitter requires graph to be UNDIRECTED"); } @@ -158,6 +182,8 @@ private EdgeSplitter.SplitResult split(IdMap sourceNodes, IdMap targetNodes, Gra trainConfig.randomSeed(), sourceNodes, targetNodes, + selectedRelType, + remainingRelType, ConcurrencyConfig.DEFAULT_CONCURRENCY ); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java index 48c0033012b..80a4b1ae7d2 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java @@ -203,17 +203,19 @@ void streamLoadedRelationshipPropertiesForTypeSubset() { @Test void streamMutatedRelationshipProperties() { + var relType = RelationshipType.of("NEW_REL"); GraphStore graphStore = GraphStoreCatalog.get(getUsername(), DatabaseId.of(db), TEST_GRAPH_SAME_PROPERTIES).graphStore(); RelationshipsBuilder relImporter = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(relType) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of("newRelProp3")) .build(); relImporter.addFromInternal(0, 1, 23D); - graphStore.addRelationshipType(RelationshipType.of("NEW_REL"), relImporter.build()); + graphStore.addRelationshipType(relType, relImporter.build()); String graphStreamQuery = formatWithLocale( "CALL gds.graph.relationshipProperties.stream(" + @@ -283,17 +285,20 @@ void streamLoadedRelationshipPropertyForTypeSubset() { @Test void streamMutatedNodeProperty() { + var relType = RelationshipType.of("NEW_REL"); + GraphStore graphStore = GraphStoreCatalog.get(getUsername(), DatabaseId.of(db), TEST_GRAPH_SAME_PROPERTIES).graphStore(); RelationshipsBuilder relImporter = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) + .relationshipType(relType) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of("newRelProp3")) .build(); relImporter.addFromInternal(0, 1, 23D); - graphStore.addRelationshipType(RelationshipType.of("NEW_REL"), relImporter.build()); + graphStore.addRelationshipType(relType, relImporter.build()); String graphStreamQuery = formatWithLocale( "CALL gds.graph.relationshipProperty.stream(" + diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java index 58a10d577e9..ed2301013a6 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java @@ -108,10 +108,14 @@ protected void updateGraphStore( Collection labelFilter = computationResult.algorithm().labelFilter().predictNodeLabels(); var graph = graphStore.getGraph(labelFilter); - var concurrency = computationResult.config().concurrency(); + var config = computationResult.config(); + var concurrency = config.concurrency(); + var mutateRelationshipType = RelationshipType.of(config.mutateRelationshipType()); + var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .aggregation(Aggregation.SINGLE) .nodes(graph) + .relationshipType(mutateRelationshipType) .orientation(Orientation.UNDIRECTED) .addPropertyConfig(GraphFactory.PropertyConfig.of(computationResult.config().mutateProperty())) .concurrency(concurrency) @@ -134,10 +138,11 @@ protected void updateGraphStore( var relationships = relationshipsBuilder.build(); - var config = computationResult.config(); + + computationResult .graphStore() - .addRelationshipType(RelationshipType.of(config.mutateRelationshipType()), relationships); + .addRelationshipType(mutateRelationshipType, relationships); resultBuilder.withRelationshipsWritten(relationships.topology().elementCount()); } }; diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java index 73cfba0b00c..07cf9ed3b9d 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java @@ -56,6 +56,7 @@ protected void updateGraphStore( var relationshipsBuilder = GraphFactory .initRelationshipsBuilder() + .relationshipType(mutateRelationshipType) .nodes(computationResult.graph()) .addPropertyConfig(GraphFactory.PropertyConfig.of(TOTAL_COST_KEY)) .orientation(Orientation.NATURAL) diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java index 77b4f5f43b6..74a3a8083b7 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java @@ -72,15 +72,15 @@ public ComputationResultConsumer computationResult, SimilarityGraphResult similarityGraphResult, String relationshipPropertyKey, @@ -129,11 +132,13 @@ private SingleTypeRelationships getRelationships( ) { SingleTypeRelationships resultRelationships; + if (similarityGraphResult.isTopKGraph()) { TopKGraph topKGraph = (TopKGraph) similarityGraphResult.similarityGraph(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(topKGraph) + .relationshipType(relationshipType) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of(relationshipPropertyKey)) .concurrency(1) @@ -167,6 +172,7 @@ private SingleTypeRelationships getRelationships( var similarityGraph = (HugeGraph) similarityGraphResult.similarityGraph(); resultRelationships = SingleTypeRelationships.of( + relationshipType, similarityGraph.relationshipTopology(), similarityGraph.schema().direction(), similarityGraph.relationshipProperties(), diff --git a/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateProc.java b/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateProc.java index 2f0b7d90f5c..e6e90b0d516 100644 --- a/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateProc.java +++ b/proc/similarity/src/main/java/org/neo4j/gds/similarity/knn/KnnMutateProc.java @@ -141,6 +141,7 @@ public ComputationResultConsumer computationResult, + RelationshipType relationshipType, ComputationResult computationResult, SimilarityGraphResult similarityGraphResult, String relationshipPropertyKey, SimilarityProc.SimilarityResultBuilder resultBuilder @@ -155,6 +158,7 @@ private SingleTypeRelationships getRelationships( RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(topKGraph) + .relationshipType(relationshipType) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of(relationshipPropertyKey)) .concurrency(1) @@ -188,6 +192,7 @@ private SingleTypeRelationships getRelationships( HugeGraph similarityGraph = (HugeGraph) similarityGraphResult.similarityGraph(); relationships = SingleTypeRelationships.of( + relationshipType, similarityGraph.relationshipTopology(), similarityGraph.schema().direction(), similarityGraph.relationshipProperties(), diff --git a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java index f54713a45e1..7cd225c750d 100644 --- a/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java +++ b/subgraph-filtering/src/main/java/org/neo4j/gds/beta/filter/RelationshipsFilter.java @@ -115,6 +115,7 @@ private static SingleTypeRelationships filterRelationshipType( var relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(outputNodes) + .relationshipType(relType) .concurrency(concurrency) .addAllPropertyConfigs(propertyConfigs) .indexInverse(graphStore.inverseIndexedRelationshipTypes().contains(relType)) diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index b9dda03539c..0e525407600 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -354,6 +354,7 @@ private Map createRelationshipBuilders( return GraphFactory.initRelationshipsBuilder() .nodes(idMap) + .relationshipType(relTypeAndProperty.getKey()) .orientation(graphProjectConfig.orientation()) .aggregation(graphProjectConfig.aggregation()) .indexInverse(graphProjectConfig.indexInverse()) From 7e2f011f340fb2fbde63e2510c51a724f60a5338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 17:18:21 +0100 Subject: [PATCH 157/400] Fix compilation after rebase --- .../neo4j/gds/beta/generator/RandomGraphGenerator.java | 4 ++-- .../java/org/neo4j/gds/core/loading/CSRGraphStore.java | 2 +- .../neo4j/gds/core/loading/RelationshipImportResult.java | 6 +++--- .../neo4j/gds/core/loading/SingleTypeRelationships.java | 9 ++++----- .../gds/core/loading/construction/GraphFactory.java | 4 ++-- .../construction/SingleTypeRelationshipsBuilder.java | 6 +++--- .../java/org/neo4j/gds/projection/GraphAggregator.java | 5 +---- .../gds/api/schema/MutableRelationshipSchemaEntry.java | 4 ++-- .../neo4j/gds/api/schema/RelationshipSchemaEntry.java | 7 +++++++ .../core/io/file/GraphStoreRelationshipVisitorTest.java | 5 ++--- 10 files changed, 27 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java index e0e01eae749..6b4d81ff0ba 100644 --- a/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java +++ b/core/src/main/java/org/neo4j/gds/beta/generator/RandomGraphGenerator.java @@ -29,7 +29,7 @@ import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.config.RandomGraphGeneratorConfig.AllowSelfLoops; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.huge.HugeGraph; @@ -145,7 +145,7 @@ public HugeGraph generate() { var relationships = relationshipsBuilder.build(); - RelationshipSchema relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipSchema.set(relationships.relationshipSchemaEntry()); var graphSchema = MutableGraphSchema.of( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java index 6fb4d930b93..6493dfa738c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java @@ -415,7 +415,7 @@ public void addRelationshipType( ) { updateGraphStore(graphStore -> { graphStore.relationships.computeIfAbsent(relationshipType, __ -> { - schema().relationshipSchema().set(relationships.relationshipSchemaEntry()); + schema.relationshipSchema().set(relationships.relationshipSchemaEntry()); return relationships; }); }); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java index 72760220ba3..46359b0c7a3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java @@ -32,7 +32,7 @@ import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.ValueTypes; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.RelationshipSchemaEntry; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.values.storable.NumberType; import java.util.Collection; @@ -59,7 +59,7 @@ static RelationshipImportResult of( topologies.forEach((relationshipType, topology) -> { Direction direction = directions.get(relationshipType); - var schemaEntry = new RelationshipSchemaEntry(relationshipType, direction); + var schemaEntry = new MutableRelationshipSchemaEntry(relationshipType, direction); relationshipImportResultBuilder.putImportResult( @@ -112,7 +112,7 @@ static RelationshipImportResult of(Collection properties(); @@ -70,7 +69,7 @@ default SingleTypeRelationships filter(String propertyKey) { RelationshipSchemaEntry entry = relationshipSchemaEntry(); - var filteredEntry = new RelationshipSchemaEntry(entry.identifier(), entry.direction()) + var filteredEntry = new MutableRelationshipSchemaEntry(entry.identifier(), entry.direction()) .addProperty(propertyKey, entry.properties().get(propertyKey)); return SingleTypeRelationships.builder() @@ -105,7 +104,7 @@ static SingleTypeRelationships of( Optional properties, Optional propertySchema ) { - var schemaEntry = new RelationshipSchemaEntry(relationshipType, direction); + var schemaEntry = new MutableRelationshipSchemaEntry(relationshipType, direction); propertySchema.ifPresent(schema -> schemaEntry.addProperty(schema.key(), schema)); return SingleTypeRelationships.builder() diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java index 161a0aa5a56..10dd36417f2 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/GraphFactory.java @@ -36,8 +36,8 @@ import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableNodeSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.NodeSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.IdMapBehaviorServiceProvider; import org.neo4j.gds.core.concurrency.Pools; @@ -309,7 +309,7 @@ public static HugeGraph create(IdMap idMap, SingleTypeRelationships relationship assert relationshipPropertyStore.values().size() == 1: "Cannot instantiate graph with more than one relationship property."; }); - RelationshipSchema relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); relationshipSchema.set(relationships.relationshipSchemaEntry()); return create( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java index 3d9d73a89f9..65ad62b0405 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java @@ -30,8 +30,8 @@ import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; import org.neo4j.gds.api.schema.ImmutableRelationshipPropertySchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.gds.api.schema.RelationshipPropertySchema; -import org.neo4j.gds.api.schema.RelationshipSchemaEntry; import org.neo4j.gds.core.compress.AdjacencyCompressor; import org.neo4j.gds.core.compress.AdjacencyListsWithProperties; import org.neo4j.gds.core.concurrency.RunWithConcurrency; @@ -152,8 +152,8 @@ SingleTypeRelationships build( return singleTypeRelationshipImportResult(); } - RelationshipSchemaEntry relationshipSchemaEntry(Optional properties) { - RelationshipSchemaEntry entry = new RelationshipSchemaEntry( + MutableRelationshipSchemaEntry relationshipSchemaEntry(Optional properties) { + var entry = new MutableRelationshipSchemaEntry( relationshipType, direction ); diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java index a7160ee3bb4..9f0d8fabf07 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/GraphAggregator.java @@ -28,9 +28,6 @@ import org.neo4j.gds.api.schema.ImmutableMutableGraphSchema; import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchema; -import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; -import org.neo4j.gds.api.schema.ImmutableGraphSchema; -import org.neo4j.gds.api.schema.RelationshipSchema; import org.neo4j.gds.compat.CompatUserAggregator; import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.ConfigKeyValidation; @@ -535,7 +532,7 @@ private void buildRelationshipsWithProperties( ) { var relationshipImportResultBuilder = RelationshipImportResult.builder(); - var relationshipSchema = RelationshipSchema.empty(); + var relationshipSchema = MutableRelationshipSchema.empty(); this.relImporters.forEach((relationshipType, relImporter) -> { var relationships = relImporter.build( Optional.of(valueMapper), diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java index 2f6826f2c63..64c12948339 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/MutableRelationshipSchemaEntry.java @@ -37,7 +37,7 @@ public class MutableRelationshipSchemaEntry implements RelationshipSchemaEntry { private final Direction direction; private final Map properties; - MutableRelationshipSchemaEntry(RelationshipType identifier, Direction direction) { + public MutableRelationshipSchemaEntry(RelationshipType identifier, Direction direction) { this(identifier, direction, new LinkedHashMap<>()); } @@ -51,7 +51,7 @@ public MutableRelationshipSchemaEntry( this.properties = properties; } - static MutableRelationshipSchemaEntry from(RelationshipSchemaEntry fromEntry) { + public static MutableRelationshipSchemaEntry from(RelationshipSchemaEntry fromEntry) { return new MutableRelationshipSchemaEntry( fromEntry.identifier(), fromEntry.direction(), diff --git a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java index 0f70faeec82..406c1d776f6 100644 --- a/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java +++ b/graph-schema-api/src/main/java/org/neo4j/gds/api/schema/RelationshipSchemaEntry.java @@ -25,4 +25,11 @@ public interface RelationshipSchemaEntry extends ElementSchemaEntry Date: Wed, 1 Feb 2023 10:21:48 +0100 Subject: [PATCH 158/400] Address PR comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../java/org/neo4j/gds/leiden/GraphAggregationPhase.java | 2 +- algo/src/main/java/org/neo4j/gds/louvain/Louvain.java | 2 +- .../org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java b/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java index a9f65e589cf..14c90a3ac52 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java @@ -148,7 +148,7 @@ Graph run() { IdMap idMap = nodesBuilder.build().idMap(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) - .relationshipType(RelationshipType.of("REL")) + .relationshipType(RelationshipType.of("_IGNORED_")) .orientation(direction.toOrientation()) .addPropertyConfig(GraphFactory.PropertyConfig.builder() .propertyKey("property") diff --git a/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java b/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java index 5c2694f2824..2754623084e 100644 --- a/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java +++ b/algo/src/main/java/org/neo4j/gds/louvain/Louvain.java @@ -234,7 +234,7 @@ private Graph summarizeGraph( IdMap idMap = nodesBuilder.build().idMap(); RelationshipsBuilder relationshipsBuilder = GraphFactory.initRelationshipsBuilder() .nodes(idMap) - .relationshipType(RelationshipType.of("REL")) + .relationshipType(RelationshipType.of("IGNORED")) .orientation(rootGraph.schema().direction().toOrientation()) .addPropertyConfig(GraphFactory.PropertyConfig.builder() .propertyKey("property") diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index ba3f0d48798..8ff376380ce 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -36,7 +36,6 @@ import org.neo4j.gds.core.loading.ImmutableStaticCapabilities; import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; -import org.neo4j.gds.core.loading.SingleTypeRelationships; import org.neo4j.gds.core.loading.construction.GraphFactory; import org.neo4j.gds.core.loading.construction.NodesBuilder; import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; @@ -234,14 +233,14 @@ private void importGraphProperties(FileInput fileInput) { } public static RelationshipImportResult relationshipImportResult(Map relationshipBuildersByType) { - Map foo = relationshipBuildersByType.entrySet() + var relationshipsByType = relationshipBuildersByType.entrySet() .stream() .collect(Collectors.toMap( e -> RelationshipType.of(e.getKey()), e -> e.getValue().build() )); - return RelationshipImportResult.builder().importResults(foo).build(); + return RelationshipImportResult.builder().importResults(relationshipsByType).build(); } @ValueClass From fa6df8f4c3aebdc51d595b67e61969945c2898fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Wed, 1 Feb 2023 10:44:55 +0100 Subject: [PATCH 159/400] Fix GraphAggregationPhaseTest --- .../java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java b/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java index 3d38d71303a..317fc5cbe52 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/GraphAggregationPhaseTest.java @@ -101,8 +101,8 @@ void testGraphAggregation() { assertGraphEquals( fromGdl( "(c1), (c2), " + - "(c1)-[:REL {w: 4.0}]->(c2), " + - "(c2)-[:REL {w: 4.0}]->(c1)" + "(c1)-[:_IGNORED_ {w: 4.0}]->(c2), " + + "(c2)-[:_IGNORED_ {w: 4.0}]->(c1)" ), aggregatedGraph ); From ff02915bfd89296849c9d10e1be76eaf5f7a63e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 14:31:43 +0100 Subject: [PATCH 160/400] Remove RelationshipType parameter from GraphStore.addRelationshipType --- .../beta/indexInverse/InverseRelationshipsTest.java | 4 ++-- .../neo4j/gds/beta/undirected/ToUndirectedTest.java | 6 +++--- .../org/neo4j/gds/beta/walking/CollapsePathTest.java | 6 +++--- .../neo4j/gds/walking/CollapsePathMutateProc.java | 6 +----- core/src/main/java/org/neo4j/gds/api/GraphStore.java | 5 +---- .../java/org/neo4j/gds/api/GraphStoreAdapter.java | 7 ++----- .../neo4j/gds/core/loading/CSRGraphStoreTest.java | 6 +++--- .../org/neo4j/gds/core/loading/GraphStoreTest.java | 1 - .../org/neo4j/gds/core/cypher/CypherGraphStore.java | 12 +++++------- .../train/LinkPredictionRelationshipSampler.java | 6 +++--- .../neo4j/gds/beta/undirected/ToUndirectedSpec.java | 5 +---- .../GraphStreamRelationshipPropertiesProcTest.java | 10 ++++------ .../predict/LinkPredictionPipelineMutateProc.java | 2 +- .../ml/splitting/SplitRelationshipsMutateProc.java | 4 ++-- .../gds/paths/ShortestPathMutateResultConsumer.java | 2 +- .../paths/spanningtree/SpanningTreeMutateSpec.java | 2 +- .../gds/paths/steiner/SteinerTreeMutateSpec.java | 2 +- .../paths/traverse/TraverseMutateResultConsumer.java | 2 +- .../filteredknn/FilteredKnnMutateProc.java | 1 - .../FilteredNodeSimilarityMutateSpec.java | 2 +- .../org/neo4j/gds/similarity/knn/KnnMutateProc.java | 1 - .../similarity/nodesim/NodeSimilarityMutateProc.java | 5 +---- 22 files changed, 37 insertions(+), 60 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/beta/indexInverse/InverseRelationshipsTest.java b/algo/src/test/java/org/neo4j/gds/beta/indexInverse/InverseRelationshipsTest.java index 1c75624deef..88ec03e6aac 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/indexInverse/InverseRelationshipsTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/indexInverse/InverseRelationshipsTest.java @@ -86,7 +86,7 @@ void shouldCreateIndexedRelationships(int concurrency) { // we need to use the same name for assertGraphEquals to work graphStore.deleteRelationships(relationshipType); - graphStore.addRelationshipType(relationshipType, inverseRelationshipsPerType.get(relationshipType)); + graphStore.addRelationshipType(inverseRelationshipsPerType.get(relationshipType)); for (String relationshipPropertyKey : inverseGraphStore.relationshipPropertyKeys()) { assertGraphEquals( @@ -118,7 +118,7 @@ void shouldIndexMultipleTypes(Object relTypes) { internalTypes.forEach(internalType -> { // we need to use the same name for assertGraphEquals to work graphStore.deleteRelationships(internalType); - graphStore.addRelationshipType(internalType, inverseRelationshipsPerType.get(internalType)); + graphStore.addRelationshipType(inverseRelationshipsPerType.get(internalType)); }); for (String relationshipPropertyKey : inverseGraphStore.relationshipPropertyKeys()) { diff --git a/algo/src/test/java/org/neo4j/gds/beta/undirected/ToUndirectedTest.java b/algo/src/test/java/org/neo4j/gds/beta/undirected/ToUndirectedTest.java index 7066237bd4d..43a57663caf 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/undirected/ToUndirectedTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/undirected/ToUndirectedTest.java @@ -82,7 +82,7 @@ void shouldCreateUndirectedRelationships(int concurrency) { Pools.DEFAULT ).compute(); - directedGraphStore.addRelationshipType(RelationshipType.of(config.mutateRelationshipType()), undirectedRelationships); + directedGraphStore.addRelationshipType(undirectedRelationships); for (String relationshipPropertyKey : undirectedGraphStore.relationshipPropertyKeys()) { assertGraphEquals( @@ -129,7 +129,7 @@ void shouldCreateUndirectedRelationshipsWithSingleRelationshipProperty(int concu Pools.DEFAULT ).compute(); - singleDirectedGraphStore.addRelationshipType(RelationshipType.of(config.mutateRelationshipType()), undirectedRelationships); + singleDirectedGraphStore.addRelationshipType(undirectedRelationships); assertGraphEquals( singleUndirectedGraphStore.getGraph(RelationshipType.of("T2"), Optional.of("prop1")), @@ -174,7 +174,7 @@ void shouldCreateUndirectedRelationshipsWithNoRelationshipProperty(int concurren Pools.DEFAULT ).compute(); - noPropertyDirectedGraphStore.addRelationshipType(RelationshipType.of(config.mutateRelationshipType()), undirectedRelationships); + noPropertyDirectedGraphStore.addRelationshipType(undirectedRelationships); assertGraphEquals( noPropertyUndirectedGraphStore.getGraph(RelationshipType.of("T2")), diff --git a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java index f97768619ab..c7499f19ff0 100644 --- a/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java +++ b/algo/src/test/java/org/neo4j/gds/beta/walking/CollapsePathTest.java @@ -158,7 +158,7 @@ void runWithDifferentRelationshipTypes() { } private void assertResultGraph(GraphStore graphStore, SingleTypeRelationships relationships, String expected) { - graphStore.addRelationshipType(RelationshipType.of("SAME_DRUG"), relationships); + graphStore.addRelationshipType(relationships); assertGraphEquals( fromGdl(expected), @@ -197,7 +197,7 @@ void shouldComputeForAllNodesWithoutNodeLabelsSpecified() { var relationships = new CollapsePathAlgorithmFactory() .build(graphStore, config, ProgressTracker.NULL_TRACKER) .compute(); - graphStore.addRelationshipType(mutateRelType, relationships); + graphStore.addRelationshipType(relationships); var resultGraph = graphStore.getGraph(mutateRelType); // then two relationships should be created @@ -222,7 +222,7 @@ void shouldComputeForSubsetOfNodesWithNodeLabelsSpecified() { var relationships = new CollapsePathAlgorithmFactory() .build(graphStore, config, ProgressTracker.NULL_TRACKER) .compute(); - graphStore.addRelationshipType(mutateRelType, relationships); + graphStore.addRelationshipType(relationships); var resultGraph = graphStore.getGraph(mutateRelType); // a single relationship is created (there is no Dog) diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/walking/CollapsePathMutateProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/walking/CollapsePathMutateProc.java index 310bb21c4ba..80f61eeabf9 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/walking/CollapsePathMutateProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/walking/CollapsePathMutateProc.java @@ -22,7 +22,6 @@ import org.neo4j.gds.GraphStoreAlgorithmFactory; import org.neo4j.gds.MutateComputationResultConsumer; import org.neo4j.gds.MutateProc; -import org.neo4j.gds.RelationshipType; import org.neo4j.gds.beta.walking.CollapsePath; import org.neo4j.gds.beta.walking.CollapsePathAlgorithmFactory; import org.neo4j.gds.beta.walking.CollapsePathConfig; @@ -72,10 +71,7 @@ protected void updateGraphStore( ComputationResult computationResult, ExecutionContext executionContext ) { - computationResult.graphStore().addRelationshipType( - RelationshipType.of(computationResult.config().mutateRelationshipType()), - computationResult.result() - ); + computationResult.graphStore().addRelationshipType(computationResult.result()); resultBuilder.withRelationshipsWritten(computationResult.result().topology().elementCount()); } diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStore.java b/core/src/main/java/org/neo4j/gds/api/GraphStore.java index a59b9e2895c..e82d712445e 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStore.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStore.java @@ -163,10 +163,7 @@ default Collection relationshipPropertyKeys(Collection RelationshipProperty relationshipPropertyValues(RelationshipType relationshipType, String propertyKey); - void addRelationshipType( - RelationshipType relationshipType, - SingleTypeRelationships relationships - ); + void addRelationshipType(SingleTypeRelationships relationships); void addInverseIndex( RelationshipType relationshipType, diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java b/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java index 80dfbdcf22f..f0f978e8323 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStoreAdapter.java @@ -230,11 +230,8 @@ public RelationshipProperty relationshipPropertyValues( } @Override - public void addRelationshipType( - RelationshipType relationshipType, - SingleTypeRelationships relationships - ) { - graphStore.addRelationshipType(relationshipType, relationships); + public void addRelationshipType(SingleTypeRelationships relationships) { + graphStore.addRelationshipType(relationships); } @Override diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java index a0ad1b2c2cc..369a0141ddd 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java @@ -74,7 +74,7 @@ void addRelationshipTypeUndirected() { relBuilder.add(gdlFactory.nodeId("a"), gdlFactory.nodeId("d")); - graphStore.addRelationshipType(RelationshipType.of("NEW"), relBuilder.build()); + graphStore.addRelationshipType(relBuilder.build()); assertThat(graphStore.relationshipCount()).isEqualTo(6); assertThat(graphStore.schema().relationshipSchema().isUndirected()).isEqualTo(true); @@ -102,7 +102,7 @@ void addRelationshipTypeDirected() { relBuilder.add(gdlFactory.nodeId("a"), gdlFactory.nodeId("d")); - graphStore.addRelationshipType(RelationshipType.of("NEW"), relBuilder.build()); + graphStore.addRelationshipType(relBuilder.build()); assertThat(graphStore.relationshipCount()).isEqualTo(3); assertThat(graphStore.schema().relationshipSchema().isUndirected()).isEqualTo(false); @@ -174,7 +174,7 @@ void addRelationshipTypeMixed(Orientation baseOrientation, int baseRelCount, Ori relBuilder.add(gdlFactory.nodeId("a"), gdlFactory.nodeId("d")); - graphStore.addRelationshipType(RelationshipType.of("NEW"), relBuilder.build()); + graphStore.addRelationshipType(relBuilder.build()); assertThat(graphStore.relationshipCount()).isEqualTo(totalRelCount); assertThat(graphStore.schema().relationshipSchema().isUndirected()).isEqualTo(false); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java index f70dfca558f..89714b13885 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreTest.java @@ -173,7 +173,6 @@ void testModificationDate() throws InterruptedException { Thread.sleep(42); RelationshipType bar = RelationshipType.of("BAR"); graphStore.addRelationshipType( - bar, SingleTypeRelationships.of( bar, ImmutableTopology.of(CompressedAdjacencyList.EMPTY, 0, false), diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/CypherGraphStore.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/CypherGraphStore.java index 3cfd28b3210..571b9e828e9 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/CypherGraphStore.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/CypherGraphStore.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.cypher; import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.GraphStoreAdapter; import org.neo4j.gds.api.IdMap; @@ -102,18 +101,17 @@ public void addNodeProperty( @Override - public void addRelationshipType( - RelationshipType relationshipType, - SingleTypeRelationships relationships - ) { - innerGraphStore().addRelationshipType(relationshipType, relationships); + public void addRelationshipType(SingleTypeRelationships relationships) { + innerGraphStore().addRelationshipType(relationships); relationships.properties().ifPresent( properties -> properties .keySet() .forEach(propertyKey -> stateVisitors.forEach(stateVisitor -> stateVisitor.relationshipPropertyAdded( propertyKey))) ); - stateVisitors.forEach(stateVisitor -> stateVisitor.relationshipTypeAdded(relationshipType.name())); + + var relType = relationships.relationshipSchemaEntry().identifier(); + stateVisitors.forEach(stateVisitor -> stateVisitor.relationshipTypeAdded(relType.name)); } public RelationshipIds relationshipIds() { diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java index 8f64f2f6644..c51c61169b7 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java @@ -156,8 +156,8 @@ public void splitAndSampleRelationships( negativeSampler.produceNegativeSamples(testSplitResult.selectedRels(), trainSplitResult.selectedRels()); // 4. Update graphStore with (positive+negative) 'TEST' and 'TRAIN' edges - graphStore.addRelationshipType(splitConfig.testRelationshipType(), testSplitResult.selectedRels().build()); - graphStore.addRelationshipType(splitConfig.trainRelationshipType(), trainSplitResult.selectedRels().build()); + graphStore.addRelationshipType(testSplitResult.selectedRels().build()); + graphStore.addRelationshipType(trainSplitResult.selectedRels().build()); validateTestSplit(graphStore); validateTrainSplit(graphStore); @@ -194,7 +194,7 @@ private EdgeSplitter.SplitResult split( ); var remainingRels = splitResult.remainingRels().build(); - graphStore.addRelationshipType(remainingRelType, remainingRels); + graphStore.addRelationshipType(remainingRels); return splitResult; } diff --git a/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java b/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java index da70a66c493..f30d9541520 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java +++ b/proc/beta/src/main/java/org/neo4j/gds/beta/undirected/ToUndirectedSpec.java @@ -79,10 +79,7 @@ protected void updateGraphStore( ComputationResult computationResult, ExecutionContext executionContext ) { - computationResult.graphStore().addRelationshipType( - RelationshipType.of(computationResult.config().mutateRelationshipType()), - computationResult.result() - ); + computationResult.graphStore().addRelationshipType(computationResult.result()); resultBuilder.withRelationshipsWritten(computationResult.result().topology().elementCount()); } }; diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java index 80a4b1ae7d2..dcabce64833 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamRelationshipPropertiesProcTest.java @@ -203,19 +203,18 @@ void streamLoadedRelationshipPropertiesForTypeSubset() { @Test void streamMutatedRelationshipProperties() { - var relType = RelationshipType.of("NEW_REL"); GraphStore graphStore = GraphStoreCatalog.get(getUsername(), DatabaseId.of(db), TEST_GRAPH_SAME_PROPERTIES).graphStore(); RelationshipsBuilder relImporter = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) - .relationshipType(relType) + .relationshipType(RelationshipType.of("NEW_REL")) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of("newRelProp3")) .build(); relImporter.addFromInternal(0, 1, 23D); - graphStore.addRelationshipType(relType, relImporter.build()); + graphStore.addRelationshipType(relImporter.build()); String graphStreamQuery = formatWithLocale( "CALL gds.graph.relationshipProperties.stream(" + @@ -285,20 +284,19 @@ void streamLoadedRelationshipPropertyForTypeSubset() { @Test void streamMutatedNodeProperty() { - var relType = RelationshipType.of("NEW_REL"); GraphStore graphStore = GraphStoreCatalog.get(getUsername(), DatabaseId.of(db), TEST_GRAPH_SAME_PROPERTIES).graphStore(); RelationshipsBuilder relImporter = GraphFactory.initRelationshipsBuilder() .nodes(graphStore.nodes()) - .relationshipType(relType) + .relationshipType(RelationshipType.of("NEW_REL")) .orientation(Orientation.NATURAL) .addPropertyConfig(GraphFactory.PropertyConfig.of("newRelProp3")) .build(); relImporter.addFromInternal(0, 1, 23D); - graphStore.addRelationshipType(relType, relImporter.build()); + graphStore.addRelationshipType(relImporter.build()); String graphStreamQuery = formatWithLocale( "CALL gds.graph.relationshipProperty.stream(" + diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java index ed2301013a6..439164dec94 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/linkmodels/pipeline/predict/LinkPredictionPipelineMutateProc.java @@ -142,7 +142,7 @@ protected void updateGraphStore( computationResult .graphStore() - .addRelationshipType(mutateRelationshipType, relationships); + .addRelationshipType(relationships); resultBuilder.withRelationshipsWritten(relationships.topology().elementCount()); } }; diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationshipsMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationshipsMutateProc.java index 134faff9f9e..eb7299f99ac 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationshipsMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationshipsMutateProc.java @@ -85,8 +85,8 @@ protected void updateGraphStore( var remainingRels = splitResult.remainingRels().build(); var config = computationResult.config(); - graphStore.addRelationshipType(config.remainingRelationshipType(), remainingRels); - graphStore.addRelationshipType(config.holdoutRelationshipType(), selectedRels); + graphStore.addRelationshipType(remainingRels); + graphStore.addRelationshipType(selectedRels); long holdoutWritten = selectedRels.topology().elementCount(); long remainingWritten = remainingRels.topology().elementCount(); diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java index 07cf9ed3b9d..3b7bcfd7fee 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathMutateResultConsumer.java @@ -76,6 +76,6 @@ protected void updateGraphStore( computationResult .graphStore() - .addRelationshipType(mutateRelationshipType, relationships); + .addRelationshipType(relationships); } } diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java index 74a3a8083b7..86d69110b5f 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/spanningtree/SpanningTreeMutateSpec.java @@ -105,7 +105,7 @@ public ComputationResultConsumer Date: Thu, 26 Jan 2023 14:41:14 +0100 Subject: [PATCH 161/400] Simplify CSRGraphStoreUtil.computeGraphSchema --- .../gds/core/loading/CSRGraphStoreUtil.java | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index c356e5d1a7a..c513727b696 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -91,7 +91,6 @@ public static CSRGraphStore createFromGraph( .relationshipSchemaEntry(entry) .topology(graph.relationshipTopology()) .properties(relationshipProperties) - .direction(direction) .build() ).build(); @@ -186,20 +185,9 @@ public static MutableGraphSchema computeGraphSchema( var nodeSchema = nodes.schema(); var relationshipSchema = MutableRelationshipSchema.empty(); - relationshipImportResult.importResults().forEach(((relationshipType, singleTypeRelationshipImportResult) -> { - relationshipSchema.getOrCreateRelationshipType( - relationshipType, - singleTypeRelationshipImportResult.direction() - ); - singleTypeRelationshipImportResult.properties() - .map(RelationshipPropertyStore::relationshipProperties) - .ifPresent(properties -> properties.forEach((propertyKey, propertyValues) -> relationshipSchema - .getOrCreateRelationshipType( - relationshipType, - singleTypeRelationshipImportResult.direction() - ) - .addProperty(propertyKey, propertyValues.propertySchema()))); - })); + relationshipImportResult + .importResults() + .forEach((__, relationships) -> relationshipSchema.set(relationships.relationshipSchemaEntry())); return MutableGraphSchema.of( nodeSchema, From 96b6f1f71d643cc48d305f05988f4161b8708507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 14:41:26 +0100 Subject: [PATCH 162/400] Remove direction from SingleTypeRelationships --- .../gds/core/loading/RelationshipImportResult.java | 1 - .../gds/core/loading/SingleTypeRelationships.java | 6 ------ .../SingleTypeRelationshipsBuilder.java | 7 ++----- .../gds/ml/splitting/DirectedEdgeSplitterTest.java | 2 +- .../ml/splitting/UndirectedEdgeSplitterTest.java | 14 +++++++------- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java index 46359b0c7a3..9b473a2d0fb 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java @@ -127,7 +127,6 @@ static RelationshipImportResult of(Collection SingleTypeRelationships .builder() - .direction(direction) .relationshipSchemaEntry(schemaEntry) ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java index df0de2ab743..e4a681e6f70 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/SingleTypeRelationships.java @@ -39,13 +39,9 @@ public interface SingleTypeRelationships { SingleTypeRelationships EMPTY = SingleTypeRelationships .builder() .relationshipSchemaEntry(new MutableRelationshipSchemaEntry(RelationshipType.of("REL"), Direction.DIRECTED)) - .direction(Direction.DIRECTED) .topology(Topology.EMPTY) .build(); - // TODO: figure out if we can remove this. - Direction direction(); - Topology topology(); MutableRelationshipSchemaEntry relationshipSchemaEntry(); @@ -75,7 +71,6 @@ default SingleTypeRelationships filter(String propertyKey) { return SingleTypeRelationships.builder() .topology(topology()) .relationshipSchemaEntry(filteredEntry) - .direction(direction()) .inverseTopology(inverseTopology()) .properties(properties) .inverseProperties(inverseProperties) @@ -108,7 +103,6 @@ static SingleTypeRelationships of( propertySchema.ifPresent(schema -> schemaEntry.addProperty(schema.key(), schema)); return SingleTypeRelationships.builder() - .direction(direction) .topology(topology) .relationshipSchemaEntry(schemaEntry) .properties( diff --git a/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java index 65ad62b0405..f6d6119b9fc 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/construction/SingleTypeRelationshipsBuilder.java @@ -264,9 +264,7 @@ SingleTypeRelationships singleTypeRelationshipImportResult() { .elementCount(relationshipCount) .build(); - var singleRelationshipTypeImportResultBuilder = SingleTypeRelationships.builder() - .topology(topology) - .direction(this.direction); + var singleRelationshipTypeImportResultBuilder = SingleTypeRelationships.builder().topology(topology); RelationshipPropertyStore properties = null; if (loadRelationshipProperty) { @@ -355,8 +353,7 @@ SingleTypeRelationships singleTypeRelationshipImportResult() { var singleRelationshipTypeImportResultBuilder = SingleTypeRelationships.builder() .topology(forwardTopology) - .inverseTopology(inverseTopology) - .direction(this.direction); + .inverseTopology(inverseTopology); RelationshipPropertyStore forwardProperties = null; if (loadRelationshipProperty) { diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java index 7b11bf164cb..025b48c97ed 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java @@ -181,7 +181,7 @@ void split() { var remainingRelationships = result.remainingRels().build(); // 2 positive selected reduces remaining assertEquals(3L, remainingRelationships.topology().elementCount()); - assertEquals(Direction.DIRECTED, remainingRelationships.direction()); + assertEquals(Direction.DIRECTED, remainingRelationships.relationshipSchemaEntry().direction()); assertFalse(remainingRelationships.topology().isMultiGraph()); assertThat(remainingRelationships.properties()).isNotEmpty(); assertRelInGraph(remainingRelationships, graph); diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java index ea044902700..690731bcad0 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java @@ -108,7 +108,7 @@ void split() { var remainingRelationships = result.remainingRels().build(); // 1 positive selected reduces remaining assertEquals(8L, remainingRelationships.topology().elementCount()); - assertEquals(Direction.UNDIRECTED, remainingRelationships.direction()); + assertEquals(Direction.UNDIRECTED, remainingRelationships.relationshipSchemaEntry().direction()); assertFalse(remainingRelationships.topology().isMultiGraph()); assertThat(remainingRelationships.properties()).isNotEmpty(); @@ -116,7 +116,7 @@ void split() { assertThat(selectedRelationships.topology()).satisfies(topology -> { assertRelSamplingProperties(selectedRelationships, graph); assertThat(topology.elementCount()).isEqualTo(1); - assertEquals(Direction.DIRECTED, selectedRelationships.direction()); + assertEquals(Direction.DIRECTED, selectedRelationships.relationshipSchemaEntry().direction()); assertFalse(topology.isMultiGraph()); }); } @@ -310,7 +310,7 @@ void splitWithFilteringWithSameSourceTargetLabels() { var remainingRelationships = result.remainingRels().build(); // 1 positive selected reduces remaining & 4 invalid relationships assertEquals(4L, remainingRelationships.topology().elementCount()); - assertEquals(Direction.UNDIRECTED, remainingRelationships.direction()); + assertEquals(Direction.UNDIRECTED, remainingRelationships.relationshipSchemaEntry().direction()); assertFalse(remainingRelationships.topology().isMultiGraph()); assertThat(remainingRelationships.properties()).isNotEmpty(); @@ -318,7 +318,7 @@ void splitWithFilteringWithSameSourceTargetLabels() { assertThat(selectedRelationships.topology()).satisfies(topology -> { assertRelSamplingProperties(selectedRelationships, graph); assertThat(topology.elementCount()).isEqualTo(1); - assertEquals(Direction.DIRECTED, selectedRelationships.direction()); + assertEquals(Direction.DIRECTED, selectedRelationships.relationshipSchemaEntry().direction()); assertFalse(topology.isMultiGraph()); }); @@ -345,7 +345,7 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { var remainingRelationships = result.remainingRels().build(); // 2 positive selected reduces remaining & 4 invalid relationships assertEquals(2L, remainingRelationships.topology().elementCount()); - assertEquals(Direction.UNDIRECTED, remainingRelationships.direction()); + assertEquals(Direction.UNDIRECTED, remainingRelationships.relationshipSchemaEntry().direction()); assertFalse(remainingRelationships.topology().isMultiGraph()); assertThat(remainingRelationships.properties()).isNotEmpty(); assertRelInGraph(remainingRelationships, multiLabelGraph); @@ -354,7 +354,7 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { assertThat(selectedRelationships.topology()).satisfies(topology -> { assertRelSamplingProperties(selectedRelationships, multiLabelGraph); assertThat(topology.elementCount()).isEqualTo(2); - assertEquals(Direction.DIRECTED, selectedRelationships.direction()); + assertEquals(Direction.DIRECTED, selectedRelationships.relationshipSchemaEntry().direction()); assertFalse(topology.isMultiGraph()); }); @@ -429,7 +429,7 @@ private boolean relationshipsAreEqual(IdMap mapping, SingleTypeRelationships r1, return false; } - if (r1.direction() != r2.direction()) { + if (r1.relationshipSchemaEntry().direction() != r2.relationshipSchemaEntry().direction()) { return false; } From ba026f9e25bd3a009dac59d21777b0819129fd23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 14:48:03 +0100 Subject: [PATCH 163/400] Inline CSRGraphStoreUtil.computeGraphSchema --- .../neo4j/gds/api/CSRGraphStoreFactory.java | 20 ++++++++++++++++-- .../gds/core/loading/CSRGraphStoreUtil.java | 21 +------------------ .../neo4j/gds/core/loading/CypherFactory.java | 11 ---------- .../neo4j/gds/core/loading/NativeFactory.java | 11 ---------- .../java/org/neo4j/gds/gdl/GdlFactory.java | 19 ----------------- 5 files changed, 19 insertions(+), 63 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java index aec5bda1535..901cc5b4f28 100644 --- a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java @@ -20,6 +20,7 @@ package org.neo4j.gds.api; import org.neo4j.gds.api.schema.MutableGraphSchema; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.loading.CSRGraphStore; @@ -29,6 +30,8 @@ import org.neo4j.gds.core.loading.RelationshipImportResult; import org.neo4j.gds.mem.MemoryUsage; +import java.util.Map; + import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; public abstract class CSRGraphStoreFactory extends GraphStoreFactory { @@ -61,8 +64,21 @@ protected void logLoadingSummary(GraphStore graphStore) { } } - protected abstract MutableGraphSchema computeGraphSchema( + protected MutableGraphSchema computeGraphSchema( Nodes nodes, RelationshipImportResult relationshipImportResult - ); + ) { + var nodeSchema = nodes.schema(); + var relationshipSchema = MutableRelationshipSchema.empty(); + + relationshipImportResult + .importResults() + .forEach((__, relationships) -> relationshipSchema.set(relationships.relationshipSchemaEntry())); + + return MutableGraphSchema.of( + nodeSchema, + relationshipSchema, + Map.of() + ); + } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index c513727b696..540e6fa37d0 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -47,6 +47,7 @@ public final class CSRGraphStoreUtil { + // TODO remove rel type? public static CSRGraphStore createFromGraph( DatabaseId databaseId, HugeGraph graph, @@ -177,25 +178,5 @@ private static Optional constructRelationshipProperti } - // TODO: remove this method - public static MutableGraphSchema computeGraphSchema( - Nodes nodes, - RelationshipImportResult relationshipImportResult - ) { - var nodeSchema = nodes.schema(); - var relationshipSchema = MutableRelationshipSchema.empty(); - - relationshipImportResult - .importResults() - .forEach((__, relationships) -> relationshipSchema.set(relationships.relationshipSchemaEntry())); - - return MutableGraphSchema.of( - nodeSchema, - relationshipSchema, - Map.of() - ); - } - - private CSRGraphStoreUtil() {} } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index de26cc597c7..d7c81c73cc7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -32,7 +32,6 @@ import org.neo4j.gds.api.CSRGraphStoreFactory; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphLoaderContext; -import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.config.GraphProjectFromCypherConfig; @@ -118,16 +117,6 @@ public GraphDimensions estimationDimensions() { .build(); } - @Override - protected MutableGraphSchema computeGraphSchema( - Nodes nodes, RelationshipImportResult relationshipImportResult - ) { - return CSRGraphStoreUtil.computeGraphSchema( - nodes, - relationshipImportResult - ); - } - @Override public CSRGraphStore build() { // Temporarily override the security context to enforce read-only access during load diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java index f82975a1d2a..4d0cd5d7a2a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java @@ -28,7 +28,6 @@ import org.neo4j.gds.api.CSRGraphStoreFactory; import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.api.IdMap; -import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.config.GraphProjectFromStoreConfig; import org.neo4j.gds.core.GraphDimensions; @@ -295,16 +294,6 @@ protected ProgressTracker initProgressTracker() { ); } - @Override - protected MutableGraphSchema computeGraphSchema( - Nodes nodes, RelationshipImportResult relationshipImportResult - ) { - return CSRGraphStoreUtil.computeGraphSchema( - nodes, - relationshipImportResult - ); - } - @Override public CSRGraphStore build() { validate(dimensions, storeConfig); diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index 0e525407600..c1aaf535646 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -31,7 +31,6 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; @@ -160,24 +159,6 @@ protected ProgressTracker initProgressTracker() { return ProgressTracker.NULL_TRACKER; } - @Override - protected MutableGraphSchema computeGraphSchema(Nodes nodes, RelationshipImportResult relationshipImportResult) { - var relationshipSchema = relationshipImportResult.importResults().values().stream().reduce( - MutableRelationshipSchema.empty(), - (unionSchema, relationships) -> { - unionSchema.set(relationships.relationshipSchemaEntry()); - return unionSchema; - }, - MutableRelationshipSchema::union - ); - - return MutableGraphSchema.of( - nodes.schema(), - relationshipSchema, - Map.of() - ); - } - @Override public CSRGraphStore build() { var nodes = loadNodes(); From eba41ac8f0c31c898cfa2ce72777d1ac10ce2c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 15:01:37 +0100 Subject: [PATCH 164/400] Set correct property schema for native projections --- .../neo4j/gds/core/loading/RelationshipImportResult.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java index 9b473a2d0fb..c90a32b0cb5 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java @@ -117,11 +117,9 @@ static RelationshipImportResult of(Collection{ - props.relationshipProperties().forEach((key, prop) -> { - schemaEntry.addProperty(key, prop.valueType(), prop.propertyState()); - }); - }); + properties.ifPresent(props -> props + .relationshipProperties() + .forEach((key, prop) -> schemaEntry.addProperty(key, prop.propertySchema()))); var importResultBuilder = builders.computeIfAbsent( importContext.relationshipType(), From 86acc1f2438f2f64652f1a90c8e785a39f66bdd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 15:12:01 +0100 Subject: [PATCH 165/400] Remove relationshipType parameter from CSRGraphStoreUtil.createFromGraph --- .../embeddings/graphsage/GraphSageTest.java | 1 - .../gds/core/loading/CSRGraphStoreUtil.java | 35 +++++++------------ .../core/loading/CSRGraphStoreUtilTest.java | 2 -- .../gds/beta/generator/GraphGenerateProc.java | 1 - .../org/neo4j/gds/catalog/GraphInfoTest.java | 1 - 5 files changed, 13 insertions(+), 27 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java index a15ed0fa801..35eb90754a2 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageTest.java @@ -117,7 +117,6 @@ void setUp() { graphStore = CSRGraphStoreUtil.createFromGraph( DatabaseId.random(), randomGraph, - "REL", Optional.of("weight"), 4 ); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index 540e6fa37d0..e37f63eca7a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -24,20 +24,15 @@ import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.Properties; -import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.RelationshipProperty; import org.neo4j.gds.api.RelationshipPropertyStore; import org.neo4j.gds.api.ValueTypes; -import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.properties.graph.GraphPropertyStore; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; -import org.neo4j.gds.api.schema.Direction; -import org.neo4j.gds.api.schema.MutableGraphSchema; -import org.neo4j.gds.api.schema.MutableNodeSchema; -import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.core.huge.HugeGraph; +import org.neo4j.gds.utils.StringJoining; import org.neo4j.values.storable.NumberType; import java.util.Map; @@ -47,20 +42,12 @@ public final class CSRGraphStoreUtil { - // TODO remove rel type? public static CSRGraphStore createFromGraph( DatabaseId databaseId, HugeGraph graph, - String relationshipTypeString, Optional relationshipPropertyKey, int concurrency ) { - var relationshipType = RelationshipType.of(relationshipTypeString); - Direction direction = graph.schema().isUndirected() ? Direction.UNDIRECTED : Direction.DIRECTED; - - var relationshipSchema = MutableRelationshipSchema.empty(); - var entry = relationshipSchema.getOrCreateRelationshipType(relationshipType, direction); - relationshipPropertyKey.ifPresent(property -> { if (!graph.hasRelationshipProperty()) { @@ -69,14 +56,19 @@ public static CSRGraphStore createFromGraph( property )); } - - entry.addProperty( - property, - ValueType.DOUBLE, - PropertyState.PERSISTENT - ); }); + var schema = graph.schema(); + var relationshipSchema = schema.relationshipSchema(); + + if (relationshipSchema.availableTypes().size() != 1) { + throw new IllegalArgumentException(formatWithLocale( + "The supplied graph has more than one relationship type: %s", + StringJoining.join(relationshipSchema.availableTypes().stream().map(e -> e.name)) + )); + } + var relationshipType = relationshipSchema.availableTypes().iterator().next(); + var nodeProperties = constructNodePropertiesFromGraph(graph); var relationshipProperties = constructRelationshipPropertiesFromGraph( @@ -89,13 +81,12 @@ public static CSRGraphStore createFromGraph( var relationshipImportResult = RelationshipImportResult.builder().putImportResult( relationshipType, SingleTypeRelationships.builder() - .relationshipSchemaEntry(entry) + .relationshipSchemaEntry(relationshipSchema.get(relationshipType)) .topology(graph.relationshipTopology()) .properties(relationshipProperties) .build() ).build(); - var schema = MutableGraphSchema.of(MutableNodeSchema.from(graph.schema().nodeSchema()), relationshipSchema, Map.of()); return new GraphStoreBuilder() .databaseId(databaseId) diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java index e7fac8785d8..b9d12d73cc0 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java @@ -56,7 +56,6 @@ void fromGraph() { var convertedGraphStore = CSRGraphStoreUtil.createFromGraph( DatabaseId.from("dummy"), (HugeGraph) graph, - "REL1", Optional.of("prop1"), 1 ); @@ -73,7 +72,6 @@ void shouldValidateRelationshipPropertyKey() { CSRGraphStoreUtil.createFromGraph( DatabaseId.from("dummy"), (HugeGraph) graph.innerGraph(), - "REL", Optional.of("prop1"), 1 ); diff --git a/proc/catalog/src/main/java/org/neo4j/gds/beta/generator/GraphGenerateProc.java b/proc/catalog/src/main/java/org/neo4j/gds/beta/generator/GraphGenerateProc.java index 225f8e6a42f..c8b56c6f717 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/beta/generator/GraphGenerateProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/beta/generator/GraphGenerateProc.java @@ -98,7 +98,6 @@ private GraphGenerationStats generateGraph( GraphStore graphStore = CSRGraphStoreUtil.createFromGraph( DatabaseId.of(databaseService), graph, - config.relationshipType().name, relationshipProperty, config.readConcurrency() ); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphInfoTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphInfoTest.java index 7c70e89d66a..788c458cdb9 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphInfoTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphInfoTest.java @@ -102,7 +102,6 @@ private GraphInfo create( var graphStore = CSRGraphStoreUtil.createFromGraph( DatabaseId.from("test"), graph, - "TY", Optional.empty(), 1 ); From 5586449e3801444cd7094aea0c512b0b7b85e08c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 26 Jan 2023 16:37:00 +0100 Subject: [PATCH 166/400] Use schema for relationship property related methods in CSRGraphStore. --- .../neo4j/gds/core/loading/CSRGraphStore.java | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java index 6493dfa738c..9d944944336 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java @@ -371,24 +371,14 @@ public boolean hasRelationshipProperty(RelationshipType relType, String property @Override public ValueType relationshipPropertyType(String propertyKey) { - return relationships - .values() - .stream() - .flatMap(relationship -> relationship.properties().stream()) - .filter(propertyStore -> propertyStore.containsKey(propertyKey)) - .map(propertyStore -> propertyStore.get(propertyKey).valueType()) - .findFirst() + return Optional.ofNullable(schema().relationshipSchema().unionProperties().get(propertyKey)) + .map(PropertySchema::valueType) .orElse(ValueType.UNKNOWN); } @Override public Set relationshipPropertyKeys() { - return relationships - .values() - .stream() - .flatMap(relationship -> relationship.properties().stream()) - .flatMap(propertyStore -> propertyStore.keySet().stream()) - .collect(Collectors.toSet()); + return schema().relationshipSchema().allProperties(); } @Override From 441a0001a55170193f7ece2476b60ad2a9d04a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Wed, 1 Feb 2023 10:40:34 +0100 Subject: [PATCH 167/400] Fix rebase problems --- .../main/java/org/neo4j/gds/core/loading/CSRGraphStore.java | 6 ++---- .../java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java | 3 ++- .../neo4j/gds/core/loading/RelationshipImportResult.java | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java index 9d944944336..e5781839b3b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStore.java @@ -400,11 +400,9 @@ public RelationshipProperty relationshipPropertyValues(RelationshipType relation } @Override - public void addRelationshipType( - RelationshipType relationshipType, SingleTypeRelationships relationships - ) { + public void addRelationshipType(SingleTypeRelationships relationships) { updateGraphStore(graphStore -> { - graphStore.relationships.computeIfAbsent(relationshipType, __ -> { + graphStore.relationships.computeIfAbsent(relationships.relationshipSchemaEntry().identifier(), __ -> { schema.relationshipSchema().set(relationships.relationshipSchemaEntry()); return relationships; }); diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index e37f63eca7a..a391965130b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -30,6 +30,7 @@ import org.neo4j.gds.api.properties.graph.GraphPropertyStore; import org.neo4j.gds.api.properties.nodes.NodeProperty; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.RelationshipPropertySchema; import org.neo4j.gds.core.huge.HugeGraph; import org.neo4j.gds.utils.StringJoining; @@ -58,7 +59,7 @@ public static CSRGraphStore createFromGraph( } }); - var schema = graph.schema(); + var schema = MutableGraphSchema.from(graph.schema()); var relationshipSchema = schema.relationshipSchema(); if (relationshipSchema.availableTypes().size() != 1) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java index c90a32b0cb5..e5850de9ff2 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java @@ -67,7 +67,6 @@ static RelationshipImportResult of( SingleTypeRelationships.builder() .topology(topology) .properties(Optional.ofNullable(properties.get(relationshipType))) - .direction(direction) .build() ); }); From f3cdc928d9ff2665fb748131625f62ebd9145e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 2 Feb 2023 11:07:01 +0100 Subject: [PATCH 168/400] Move relationship schema creation into RelationshipImportResult MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../neo4j/gds/api/CSRGraphStoreFactory.java | 27 +++++-------------- .../loading/RelationshipImportResult.java | 11 ++++++++ .../java/org/neo4j/gds/gdl/GdlFactory.java | 7 ++++- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java index 901cc5b4f28..bc07eeac82a 100644 --- a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java @@ -20,7 +20,6 @@ package org.neo4j.gds.api; import org.neo4j.gds.api.schema.MutableGraphSchema; -import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.loading.CSRGraphStore; @@ -46,10 +45,16 @@ public CSRGraphStoreFactory( } protected CSRGraphStore createGraphStore(Nodes nodes, RelationshipImportResult relationshipImportResult) { + var schema = MutableGraphSchema.of( + nodes.schema(), + relationshipImportResult.relationshipSchema(), + Map.of() + ); + return new GraphStoreBuilder() .databaseId(DatabaseId.of(loadingContext.graphDatabaseService())) .capabilities(capabilities) - .schema(computeGraphSchema(nodes, relationshipImportResult)) + .schema(schema) .nodes(nodes) .relationshipImportResult(relationshipImportResult) .concurrency(graphProjectConfig.readConcurrency()) @@ -63,22 +68,4 @@ protected void logLoadingSummary(GraphStore graphStore) { progressTracker.logInfo(formatWithLocale("Actual memory usage of the loaded graph: %s", memoryUsage)); } } - - protected MutableGraphSchema computeGraphSchema( - Nodes nodes, - RelationshipImportResult relationshipImportResult - ) { - var nodeSchema = nodes.schema(); - var relationshipSchema = MutableRelationshipSchema.empty(); - - relationshipImportResult - .importResults() - .forEach((__, relationships) -> relationshipSchema.set(relationships.relationshipSchemaEntry())); - - return MutableGraphSchema.of( - nodeSchema, - relationshipSchema, - Map.of() - ); - } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java index e5850de9ff2..6343b748007 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/RelationshipImportResult.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.loading; +import org.immutables.value.Value; import org.neo4j.gds.PropertyMappings; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; @@ -32,6 +33,7 @@ import org.neo4j.gds.api.Topology; import org.neo4j.gds.api.ValueTypes; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchemaEntry; import org.neo4j.values.storable.NumberType; @@ -46,6 +48,15 @@ public interface RelationshipImportResult { Map importResults(); + @Value.Lazy + default MutableRelationshipSchema relationshipSchema() { + var relationshipSchema = MutableRelationshipSchema.empty(); + + importResults().forEach((__, relationships) -> relationshipSchema.set(relationships.relationshipSchemaEntry())); + + return relationshipSchema; + } + static ImmutableRelationshipImportResult.Builder builder() { return ImmutableRelationshipImportResult.builder(); } diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index c1aaf535646..dbeab231818 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -31,6 +31,7 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.nodeproperties.ValueType; import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.api.schema.MutableGraphSchema; import org.neo4j.gds.api.schema.MutableRelationshipSchema; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.ImmutableGraphDimensions; @@ -163,7 +164,11 @@ protected ProgressTracker initProgressTracker() { public CSRGraphStore build() { var nodes = loadNodes(); var relationshipImportResult = loadRelationships(nodes.idMap()); - var schema = computeGraphSchema(nodes, relationshipImportResult); + var schema = MutableGraphSchema.of( + nodes.schema(), + relationshipImportResult.relationshipSchema(), + Map.of() + ); return new GraphStoreBuilder() .databaseId(databaseId) From 79f54531f98990f92d53d954ad5dfacc9d5c7189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Kie=C3=9Fling?= Date: Thu, 2 Feb 2023 11:07:33 +0100 Subject: [PATCH 169/400] Allow graphs without relationships in CSRGraphStoreUtil MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sören Reichardt --- .../gds/core/loading/CSRGraphStoreUtil.java | 41 +++++++++++-------- .../core/loading/CSRGraphStoreUtilTest.java | 22 ++++++++++ 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java index a391965130b..769f50b8c64 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CSRGraphStoreUtil.java @@ -62,32 +62,37 @@ public static CSRGraphStore createFromGraph( var schema = MutableGraphSchema.from(graph.schema()); var relationshipSchema = schema.relationshipSchema(); - if (relationshipSchema.availableTypes().size() != 1) { + if (relationshipSchema.availableTypes().size() > 1) { throw new IllegalArgumentException(formatWithLocale( "The supplied graph has more than one relationship type: %s", StringJoining.join(relationshipSchema.availableTypes().stream().map(e -> e.name)) )); } - var relationshipType = relationshipSchema.availableTypes().iterator().next(); var nodeProperties = constructNodePropertiesFromGraph(graph); - var relationshipProperties = constructRelationshipPropertiesFromGraph( - graph, - relationshipType, - relationshipPropertyKey, - graph.relationshipProperties() - ); - - var relationshipImportResult = RelationshipImportResult.builder().putImportResult( - relationshipType, - SingleTypeRelationships.builder() - .relationshipSchemaEntry(relationshipSchema.get(relationshipType)) - .topology(graph.relationshipTopology()) - .properties(relationshipProperties) - .build() - ).build(); - + RelationshipImportResult relationshipImportResult; + if (relationshipSchema.availableTypes().isEmpty()) { + relationshipImportResult = RelationshipImportResult.builder().build(); + } else { + var relationshipType = relationshipSchema.availableTypes().iterator().next(); + + var relationshipProperties = constructRelationshipPropertiesFromGraph( + graph, + relationshipType, + relationshipPropertyKey, + graph.relationshipProperties() + ); + + relationshipImportResult = RelationshipImportResult.builder().putImportResult( + relationshipType, + SingleTypeRelationships.builder() + .relationshipSchemaEntry(relationshipSchema.get(relationshipType)) + .topology(graph.relationshipTopology()) + .properties(relationshipProperties) + .build() + ).build(); + } return new GraphStoreBuilder() .databaseId(databaseId) diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java index b9d12d73cc0..84129d978dd 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreUtilTest.java @@ -64,6 +64,28 @@ void fromGraph() { assertGraphEquals(graphStore.getUnion(), convertedGraphStore.getUnion()); } + @Test + void fromGraphWithNoRelationships() { + String GDL = + " CREATE" + + " (a:A { foo: 42, bar: 1337 })" + + ", (b:A { foo: 84, bar: 1234 })" + + ", (c:B { foo: 23 })"; + + var gdlGraphStore = TestSupport.graphStoreFromGDL(GDL); + var gdlGraph = gdlGraphStore.getUnion(); + + var convertedGraphStore = CSRGraphStoreUtil.createFromGraph( + DatabaseId.from("dummy"), + (HugeGraph) gdlGraph, + Optional.empty(), + 1 + ); + + assertThat(convertedGraphStore.schema()).isEqualTo(gdlGraphStore.schema()); + assertGraphEquals(gdlGraphStore.getUnion(), convertedGraphStore.getUnion()); + } + @Test void shouldValidateRelationshipPropertyKey() { var graph = TestSupport.fromGdl("()-[:REL]->()"); From 08c124620574aa37ddf6fc492d9009aa7945d002 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Feb 2023 16:57:26 +0100 Subject: [PATCH 170/400] Redirects --- doc/modules/ROOT/pages/algorithms/kmeans.adoc | 1 + doc/modules/ROOT/pages/algorithms/leiden.adoc | 2 ++ .../ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc | 1 + 3 files changed, 4 insertions(+) diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 78e4bfe1627..8602b519735 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -1,3 +1,4 @@ +:page-aliases: algorithms/alpha/kmeans.adoc [[algorithms-k-means]] [.beta] = K-Means Clustering diff --git a/doc/modules/ROOT/pages/algorithms/leiden.adoc b/doc/modules/ROOT/pages/algorithms/leiden.adoc index 8180b0b1185..0c903b4a5a8 100644 --- a/doc/modules/ROOT/pages/algorithms/leiden.adoc +++ b/doc/modules/ROOT/pages/algorithms/leiden.adoc @@ -1,3 +1,5 @@ +:page-aliases: algorithms/alpha/leiden.adoc + [[algorithms-leiden]] [.beta] = Leiden diff --git a/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc index 06b95851c27..47823c3b027 100644 --- a/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc @@ -1,3 +1,4 @@ +:page-aliases: alpha-algorithms/minimum-weight-spanning-tree.adoc [[algorithms-minimum-weight-spanning-tree]] [.beta] = Minimum Weight Spanning Tree From 0b71d76bf2e8f45fbbb0f7ac9f5566a6cc0eb796 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 7 Feb 2023 13:25:02 +0000 Subject: [PATCH 171/400] Add redirects for production deployment pages Co-authored-by: Ioannis Panagiotas --- doc/modules/ROOT/pages/production-deployment/composite.adoc | 2 ++ doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/doc/modules/ROOT/pages/production-deployment/composite.adoc b/doc/modules/ROOT/pages/production-deployment/composite.adoc index 3fc171774b0..91a31a605c5 100644 --- a/doc/modules/ROOT/pages/production-deployment/composite.adoc +++ b/doc/modules/ROOT/pages/production-deployment/composite.adoc @@ -1,3 +1,5 @@ +:page-aliases: production-deployment/fabric + [[composite]] // Putting "Fabric" in the header might help with searching for the // case that users are not familiar with composite databases yet diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 39fde81e854..a1fb3c558e7 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -1,3 +1,5 @@ +:page-aliases: production-deployment/causal-cluster + [.enterprise-edition] [[cluster]] = GDS with Neo4j cluster From ac297dff717661c4245e616cee9c06ba7d47f7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 7 Feb 2023 15:09:50 +0100 Subject: [PATCH 172/400] Fix Neo4jVersion and test Co-Authored-By: Yuval Rotenberg --- .../src/main/java/org/neo4j/gds/compat/Neo4jVersion.java | 2 ++ .../src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index ba4140d6624..aa9473fe673 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -50,6 +50,8 @@ public String toString() { return "5.4"; case V_5_5: return "5.5"; + case V_RC: + return "rc"; default: throw new IllegalArgumentException("Unexpected value: " + this.name() + " (sad java 😞)"); } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index bab22d96e7f..5bfa7173d4a 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -45,7 +45,8 @@ class Neo4jVersionTest { "5.2.0, V_5_2", "5.3.0, V_5_3", "5.4.0, V_5_4", - "5.5.0, V_RC", + "5.5.0, V_5_0", + "5.6.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); From dd0d3cd19e88c9ec30fa5fd43c5ab2f810e0f569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 7 Feb 2023 16:29:37 +0100 Subject: [PATCH 173/400] Fix tests --- .../test/java/org/neo4j/gds/compat/Neo4jVersionTest.java | 3 ++- .../src/test/java/org/neo4j/gds/SysInfoProcTest.java | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 5bfa7173d4a..0ee6b822964 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -45,7 +45,7 @@ class Neo4jVersionTest { "5.2.0, V_5_2", "5.3.0, V_5_3", "5.4.0, V_5_4", - "5.5.0, V_5_0", + "5.5.0, V_5_5", "5.6.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { @@ -83,6 +83,7 @@ void shouldNotRespectVersionOverride() { "5.2.0, 5, 2", "5.3.0, 5, 3", "5.4.0, 5, 4", + "5.5.0, 5, 5", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index b43b34c7613..d07b22c5487 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -71,10 +71,10 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.4", "Neo4j Settings 5.4 (placeholder)", - "Neo4j DEV", - "Neo4j DEV (placeholder)", - "Neo4j Settings DEV", - "Neo4j Settings DEV (placeholder)", + "Neo4j 5.5", + "Neo4j 5.5 (placeholder)", + "Neo4j Settings 5.5", + "Neo4j Settings 5.5 (placeholder)", "Neo4j RC", "Neo4j RC (placeholder)", From d497cadec7354388841cc7c6ce5198e2bfc8f4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 7 Feb 2023 17:57:18 +0100 Subject: [PATCH 174/400] Publish 5.5 copmat --- .../5.5/neo4j-kernel-adapter/build.gradle | 63 ++ .../gds/compat/_55/Neo4jProxyFactoryImpl.java | 44 + .../compat/_55/SettingProxyFactoryImpl.java | 44 + .../compat/_55/BoltTransactionRunnerImpl.java | 98 ++ .../gds/compat/_55/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_55/CompatAccessModeImpl.java | 44 + .../_55/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_55/CompatIndexQueryImpl.java | 31 + .../_55/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_55/CompositeNodeCursorImpl.java | 32 + .../gds/compat/_55/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../gds/compat/_55/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_55/Neo4jProxyImpl.java | 924 ++++++++++++++++++ .../compat/_55/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_55/PartitionedStoreScan.java | 58 ++ .../_55/ReferencePropertyReference.java | 49 + .../compat/_55/ScanBasedStoreScanImpl.java | 40 + .../compat/_55/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_55/SettingProxyImpl.java | 87 ++ .../org/neo4j/gds/compat/_55/TestLogImpl.java | 146 +++ .../compat/_55/VirtualRelationshipImpl.java | 41 + .../5.5/storage-engine-adapter/build.gradle | 65 ++ .../_55/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 106 ++ .../compat/_55/InMemoryCountsStoreImpl.java | 110 +++ .../_55/InMemoryMetaDataProviderImpl.java | 190 ++++ .../gds/compat/_55/InMemoryNodeCursor.java | 82 ++ .../_55/InMemoryNodePropertyCursor.java | 45 + .../compat/_55/InMemoryPropertyCursor.java | 71 ++ .../_55/InMemoryPropertySelectionImpl.java | 55 ++ .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_55/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_55/InMemoryStorageEngineFactory.java | 550 +++++++++++ .../compat/_55/InMemoryStorageEngineImpl.java | 294 ++++++ .../compat/_55/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_55/InMemoryStoreVersion.java | 68 ++ .../_55/InMemoryTransactionIdStoreImpl.java | 75 ++ .../gds/compat/_55/InMemoryVersionCheck.java | 61 ++ .../_55/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_55/StorageEngineProxyImpl.java | 156 +++ .../InMemoryLogVersionRepository.java | 71 ++ .../InMemoryStorageCommandReaderFactory.java | 43 + .../InMemoryStorageReader55.java | 325 ++++++ 46 files changed, 4909 insertions(+) create mode 100644 compatibility/5.5/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableProcedureImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatAccessModeImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatIndexQueryImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/PartitionedStoreScan.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ReferencePropertyReference.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/TestLogImpl.java create mode 100644 compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/VirtualRelationshipImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodeCursor.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertyCursor.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStoreVersion.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryVersionCheck.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader55.java diff --git a/compatibility/5.5/neo4j-kernel-adapter/build.gradle b/compatibility/5.5/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..6eed2158956 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.5' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.5' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.5' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.5' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.5' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.5' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.5' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..931a383b46e --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.5 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.5 (placeholder)"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..78e1b894771 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.5 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.5 (placeholder)"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/BoltTransactionRunnerImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..b96e77e2fe3 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/BoltTransactionRunnerImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.bookmark.Bookmark; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.transaction.result.AdaptingBoltQuerySubscriber; +import org.neo4j.bolt.protocol.v41.message.request.RoutingContext; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new AdaptingBoltQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSucceeded(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.queryStatistics(); + } + + @Override + public AdaptingBoltQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + AdaptingBoltQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableProcedureImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableProcedureImpl.java new file mode 100644 index 00000000000..72c0b6545ed --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableUserAggregationFunctionImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..4156ab4cc4e --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatAccessModeImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatAccessModeImpl.java new file mode 100644 index 00000000000..ce96cc3a72a --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatGraphDatabaseAPIImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..0af8537b5a8 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatIndexQueryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..8cad95ee9b5 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatUsernameAuthSubjectImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..148fa614366 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompositeNodeCursorImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..8ee3052f38e --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseLayoutImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..c1dc37d80ef --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..13fdae4cce6 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..7f13c7ebe56 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_5; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.5"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java new file mode 100644 index 00000000000..2bb6641c6ea --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -0,0 +1,924 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Group; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public long pageCacheMemory() { + return pageCacheMemory.orElseGet(Configuration.super::pageCacheMemory); + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = new Group(0, null, null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor() { + var globalGroup = new Group(0, null, null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/NodeLabelIndexLookupImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..d4b58ae0e40 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/PartitionedStoreScan.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/PartitionedStoreScan.java new file mode 100644 index 00000000000..98463b3150b --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ReferencePropertyReference.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ReferencePropertyReference.java new file mode 100644 index 00000000000..afaaa02f22e --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ScanBasedStoreScanImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..28784106272 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..0b4c48c565c --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_5; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.5"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyImpl.java new file mode 100644 index 00000000000..62d6597d682 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/TestLogImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/TestLogImpl.java new file mode 100644 index 00000000000..8cc7dec1f3b --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/VirtualRelationshipImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..0e6e2b491f3 --- /dev/null +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/build.gradle b/compatibility/5.5/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..57c683e1fd5 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.5' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.5' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.5' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.5' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.5' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.5' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..0c98e19eeb7 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.5"; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCommandCreationContextImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..ac20edc223e --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.LATEST; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCountsStoreImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..8e6544b2055 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryCountsStoreImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..8762b580155 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + byteOffset, + logVersion + ); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } + + TransactionIdStore transactionIdStore() { + return this.transactionIdStore; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodeCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodeCursor.java new file mode 100644 index 00000000000..d6118867135 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodePropertyCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..3718d2ce791 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertyCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..48434b8e0a1 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertySelectionImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..146d3613887 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipPropertyCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..93fa5f6ba74 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipScanCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..42b15b04014 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipTraversalCursor.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..66cb205b6f1 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..7cfe623636e --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -0,0 +1,550 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-55"; + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker databaseReadOnlyChecker, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..1573e72d1d5 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineImpl.java @@ -0,0 +1,294 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader55; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private final StorageEngineIndexingBehaviour indexingBehaviour = () -> false; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return indexingBehaviour; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader55(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageLocksImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..122e557284b --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStoreVersion.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStoreVersion.java new file mode 100644 index 00000000000..000361016cf --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..465e50cba08 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4] + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + } + + @Override + protected CursorContext getEmptyCursorContext() { + return CursorContext.NULL_CONTEXT; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryVersionCheck.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryVersionCheck.java new file mode 100644 index 00000000000..1ae0d5d7f76 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._55.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..8c9a25a65ac --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_5; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.5"; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..7124a0829d1 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader55; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader55(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java new file mode 100644 index 00000000000..b491fbc7f32 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository() { + this(0,0); + } + + private InMemoryLogVersionRepository(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java new file mode 100644 index 00000000000..620a8aac11c --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader55.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader55.java new file mode 100644 index 00000000000..67fd5e0c840 --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader55.java @@ -0,0 +1,325 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._55.InMemoryNodeCursor; +import org.neo4j.gds.compat._55.InMemoryPropertyCursor; +import org.neo4j.gds.compat._55.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._55.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader55 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader55( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} From ee5571e664b8e5b2adf7f5e4f8828e5cc6e1d66d Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 8 Feb 2023 04:49:21 +0000 Subject: [PATCH 175/400] Log warn instead of error --- .../progress/tasks/TaskProgressTracker.java | 2 +- .../tasks/TaskProgressTrackerTest.java | 34 ++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java index 7668569cb45..eb752cb77ea 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java @@ -77,7 +77,7 @@ public TaskProgressTracker( AtomicBoolean didLog = new AtomicBoolean(false); this.onError = () -> { if (!didLog.get()) { - taskProgressLogger.logError("Tried to log progress, but there are no running tasks being tracked"); + taskProgressLogger.logWarning(":: Tried to log progress, but there are no running tasks being tracked"); didLog.set(true); } }; diff --git a/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java b/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java index 3bec5f88e5b..a593b666fc4 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.utils.progress.tasks; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; @@ -27,13 +28,17 @@ import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistry; import org.neo4j.gds.core.utils.progress.TaskStore; +import org.neo4j.gds.utils.GdsFeatureToggles; import org.neo4j.logging.Log; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.neo4j.gds.assertj.Extractors.removingThreadId; +import static org.neo4j.gds.compat.TestLog.WARN; public class TaskProgressTrackerTest { @@ -72,14 +77,33 @@ void shouldStepThroughSubtasks() { } @Test - void shouldThrowIfEndMoreTasksThanStarted() { + void shouldThrowIfEndMoreTasksThanStartedAndFeatureToggleIsEnabled() { + GdsFeatureToggles.THROW_WHEN_USING_PROGRESS_TRACKER_WITHOUT_TASKS.enableAndRun(() -> { + var task = Tasks.leaf("leaf"); + TaskProgressTracker progressTracker = progressTracker(task); + progressTracker.beginSubTask(); + progressTracker.endSubTask(); + assertThatThrownBy(progressTracker::endSubTask) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Tried to log progress, but there are no running tasks being tracked"); + }); + } + + @Test + void shouldNotThrowIfEndMoreTasksThanStarted() { var task = Tasks.leaf("leaf"); - TaskProgressTracker progressTracker = progressTracker(task); + var log = Neo4jProxy.testLog(); + var progressTracker = new TaskProgressTracker(task, log, 1, EmptyTaskRegistryFactory.INSTANCE); progressTracker.beginSubTask(); progressTracker.endSubTask(); - assertThatThrownBy(progressTracker::endSubTask) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Tried to log progress, but there are no running tasks being tracked"); + assertThatNoException() + .as("When `THROW_WHEN_USING_PROGRESS_TRACKER_WITHOUT_TASKS` is disabled (default state) we should not throw an exception.") + .isThrownBy(progressTracker::endSubTask); + + Assertions + .assertThat(log.getMessages(WARN)) + .extracting(removingThreadId()) + .containsExactly("leaf :: Tried to log progress, but there are no running tasks being tracked"); } @Test From 5c321ece0c3101656d2a83ce82d1232109912b09 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 8 Feb 2023 05:12:26 +0000 Subject: [PATCH 176/400] Explicitly disable throwing in test --- .../tasks/TaskProgressTrackerTest.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java b/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java index a593b666fc4..3c6a4ec5547 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTrackerTest.java @@ -40,7 +40,7 @@ import static org.neo4j.gds.assertj.Extractors.removingThreadId; import static org.neo4j.gds.compat.TestLog.WARN; -public class TaskProgressTrackerTest { +class TaskProgressTrackerTest { @Test void shouldStepThroughSubtasks() { @@ -91,19 +91,21 @@ void shouldThrowIfEndMoreTasksThanStartedAndFeatureToggleIsEnabled() { @Test void shouldNotThrowIfEndMoreTasksThanStarted() { - var task = Tasks.leaf("leaf"); - var log = Neo4jProxy.testLog(); - var progressTracker = new TaskProgressTracker(task, log, 1, EmptyTaskRegistryFactory.INSTANCE); - progressTracker.beginSubTask(); - progressTracker.endSubTask(); - assertThatNoException() - .as("When `THROW_WHEN_USING_PROGRESS_TRACKER_WITHOUT_TASKS` is disabled (default state) we should not throw an exception.") - .isThrownBy(progressTracker::endSubTask); - - Assertions - .assertThat(log.getMessages(WARN)) - .extracting(removingThreadId()) - .containsExactly("leaf :: Tried to log progress, but there are no running tasks being tracked"); + GdsFeatureToggles.THROW_WHEN_USING_PROGRESS_TRACKER_WITHOUT_TASKS.disableAndRun(() -> { + var task = Tasks.leaf("leaf"); + var log = Neo4jProxy.testLog(); + var progressTracker = new TaskProgressTracker(task, log, 1, EmptyTaskRegistryFactory.INSTANCE); + progressTracker.beginSubTask(); + progressTracker.endSubTask(); + assertThatNoException() + .as("When `THROW_WHEN_USING_PROGRESS_TRACKER_WITHOUT_TASKS` is disabled (default state) we should not throw an exception.") + .isThrownBy(progressTracker::endSubTask); + + Assertions + .assertThat(log.getMessages(WARN)) + .extracting(removingThreadId()) + .containsExactly("leaf :: Tried to log progress, but there are no running tasks being tracked"); + }); } @Test From f6021277614ef5406faeab3135eebb7bcdbd1db8 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 1 Feb 2023 07:04:32 +0000 Subject: [PATCH 177/400] Create page(s) listing the GDS confg settings --- doc/modules/ROOT/content-nav.adoc | 2 + .../operations-reference/appendix-a.adoc | 1 + .../configuration-settings.adoc | 140 ++++++++++++++++++ .../configuration-settings.adoc | 115 ++++++++++++++ .../pages/production-deployment/index.adoc | 1 + 5 files changed, 259 insertions(+) create mode 100644 doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc create mode 100644 doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 34790329a1e..713832e484b 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -148,6 +148,7 @@ ** xref:production-deployment/transaction-handling.adoc[] ** xref:production-deployment/composite.adoc[] ** xref:production-deployment/neo4j-cluster.adoc[] +** xref:production-deployment/configuration-settings.adoc[] ** xref:production-deployment/feature-toggles.adoc[] * xref:python-client/index.adoc[] * link:https://neo4j.com/docs/bloom-user-guide/current/bloom-tutorial/gds-integration/[Bloom visualization] @@ -157,6 +158,7 @@ *** xref:operations-reference/algorithm-references.adoc[] *** xref:operations-reference/machine-learning-references.adoc[] *** xref:operations-reference/additional-operation-references.adoc[] +*** xref:operations-reference/configuration-settings.adoc[] ** xref:appendix-b/index.adoc[] *** xref:appendix-b/migration-algos-common.adoc[] *** xref:appendix-b/migration-graph-projection.adoc[] diff --git a/doc/modules/ROOT/pages/operations-reference/appendix-a.adoc b/doc/modules/ROOT/pages/operations-reference/appendix-a.adoc index 6a3556b5c5d..184abf3dede 100644 --- a/doc/modules/ROOT/pages/operations-reference/appendix-a.adoc +++ b/doc/modules/ROOT/pages/operations-reference/appendix-a.adoc @@ -11,3 +11,4 @@ This chapter contains a full listing of all operations in the Neo4j Graph Data S * xref:operations-reference/algorithm-references.adoc[Graph Algorithms] * xref:operations-reference/machine-learning-references.adoc[Machine Learning] * xref:operations-reference/additional-operation-references.adoc[Additional Operations] +* xref:operations-reference/configuration-settings.adoc[Configuration Settings] diff --git a/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc new file mode 100644 index 00000000000..f90c28614d6 --- /dev/null +++ b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc @@ -0,0 +1,140 @@ +[[configuration-settings1]] += Configuration Settings +:description: This section describes the available configuration settings in the Neo4j Graph Data Science library. + +This page describes the available configuration settings in GDS. +Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4j-conf/#neo4j-conf[neo4j.conf] file for details on how to use configuration settings. + + +All settings + +* <>: label:enterprise-edition[Enterprise Edition] The maximum time to wait for the next command before aborting the import process. +* <>: label:enterprise-edition[Enterprise Edition] Address that clients should use to connect to the GDS Arrow Flight Server. +* <>: label:enterprise-edition[Enterprise Edition] The batch size used for arrow property export. +* <>: label:enterprise-edition[Enterprise Edition] Enable the GDS Arrow Flight Server. +* <>: label:enterprise-edition[Enterprise Edition] Never activate server-side encryption for the GDS Arrow Flight Server. +* <>: label:enterprise-edition[Enterprise Edition] Address the GDS Arrow Flight Server should bind to. +* <>: label:enterprise-edition[Enterprise Edition] Set the maximum transaction size for GDS on Read Replica +* <>: label:enterprise-edition[Enterprise Edition] Set the minimum transaction size for GDS on Read Replica +* <>: Sets the location of the file that contains the Neo4j Graph Data Science library license key +* <>: Sets the export location for file based exports. +* <>: label:enterprise-edition[Enterprise Edition] Sets the location where persisted models are stored. +* <>: Enable progress logging tracking. +* <>: Use maximum memory estimation in procedure memory guard. + + +[[gds.arrow.abortion_timeout]] +`gds.arrow.abortion_timeout` label:enterprise-edition[Enterprise Edition] + +* The maximum time to wait for the next command before aborting the import process. +* defaultValue: `10m` +* validValues: a duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`) +* dynamic: `false` + + +[[gds.arrow.advertised_listen_address]] +`gds.arrow.advertised_listen_address` label:enterprise-edition[Enterprise Edition] + +* Address that clients should use to connect to the GDS Arrow Flight Server. +* defaultValue: `:8491` +* validValues: a socket address in the format `hostname:port`, `hostname` or `:port`. If missing port or hostname it is acquired from `gds.arrow.listen_address` +* dynamic: `false` + + +[[gds.arrow.batch_size]] +`gds.arrow.batch_size` label:enterprise-edition[Enterprise Edition] + +* The batch size used for arrow property export. +* defaultValue: `10000` +* validValues: an integer +* dynamic: `true` + + +[[gds.arrow.enabled]] +`gds.arrow.enabled` label:enterprise-edition[Enterprise Edition] + +* Enable the GDS Arrow Flight Server. +* defaultValue: `false` +* validValues: a boolean +* dynamic: `false` + + +[[gds.arrow.encryption.never]] +`gds.arrow.encryption.never` label:enterprise-edition[Enterprise Edition] + +* Never activate server-side encryption for the GDS Arrow Flight Server. +* defaultValue: `false` +* validValues: a boolean +* dynamic: `false` + + +[[gds.arrow.listen_address]] +`gds.arrow.listen_address` label:enterprise-edition[Enterprise Edition] + +* The maximum time to wait for the next command before aborting the import process. +* defaultValue: `10m` +* validValues: a duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`) +* dynamic: `false` + + +[[gds.cluster.tx.max.size]] +`gds.cluster.tx.max.size` label:enterprise-edition[Enterprise Edition] + +* Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +* defaultValue: `100000` +* validValues: an integer, must be set greater than or equal to the value of `gds.cluster.tx.min.size` +* dynamic: `false` + + +[[gds.cluster.tx.min.size]] +`gds.cluster.tx.min.size` label:enterprise-edition[Enterprise Edition] + +* Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +* defaultValue: `10000` +* validValues: an integer +* dynamic: `false` + + +[[gds.enterprise.license_file]] +`gds.enterprise.license_file` + +* Sets the location of the file that contains the Neo4j Graph Data Science library license key. +* defaultValue: `No Value` +* validValues: an absolute path +* dynamic: `false` + + +[[gds.export.location]] +`gds.export.location` + +* Sets the export location for file based exports. +* defaultValue: `No Value` +* validValues: an absolute path +* dynamic: `false` + + +[[gds.model.store_location]] +`gds.model.store_location` label:enterprise-edition[Enterprise Edition] + +* Sets the location where persisted models are stored. +* defaultValue: `No Value` +* validValues: an absolute path +* dynamic: `false` + + +[[gds.progress_tracking_enabled]] +`gds.progress_tracking_enabled` + +* Enable progress logging tracking. +* defaultValue: `true` +* validValues: a boolean +* dynamic: `false` + + +[[gds.validate_using_max_memory_estimation]] +`gds.validate_using_max_memory_estimation` + +* Use maximum memory estimation in procedure memory guard. +* defaultValue: `false` +* validValues: a boolean +* dynamic: `false` diff --git a/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc new file mode 100644 index 00000000000..0eccf5a1676 --- /dev/null +++ b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc @@ -0,0 +1,115 @@ +[[configuration-settings]] += GDS Configuration Settings +:description: This section describes the available configuration settings in the Neo4j Graph Data Science library. + + +This page describes the available configuration settings in GDS. +Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4j-conf/#neo4j-conf[neo4j.conf] file for details on how to use configuration settings. + + +[.enterprise-edition] +== GDS and Arrow + +`gds.arrow.abortion_timeout` + +* The maximum time to wait for the next command before aborting the import process. +* defaultValue: `10m` +* validValues: a duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`) +* dynamic: `false` + +`gds.arrow.advertised_listen_address` + +* Address that clients should use to connect to the GDS Arrow Flight Server. +* defaultValue: `:8491` +* validValues: a socket address in the format `hostname:port`, `hostname` or `:port`. If missing port or hostname it is acquired from `gds.arrow.listen_address` +* dynamic: `false` + +`gds.arrow.batch_size` + +* The batch size used for arrow property export. +* defaultValue: `10000` +* validValues: an integer +* dynamic: `true` + +`gds.arrow.enabled` + +* Enable the GDS Arrow Flight Server. +* defaultValue: `false` +* validValues: a boolean +* dynamic: `false` + +`gds.arrow.encryption.never` + +* Never activate server-side encryption for the GDS Arrow Flight Server. +* defaultValue: `false` +* validValues: a boolean +* dynamic: `false` + +`gds.arrow.listen_address` + +* The maximum time to wait for the next command before aborting the import process. +* defaultValue: `10m` +* validValues: a duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`) +* dynamic: `false` + + +[.enterprise-edition] +== Neo4j Cluster + +`gds.cluster.tx.max.size` + +* Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +* defaultValue: `100000` +* validValues: an integer, must be set greater than or equal to the value of `gds.cluster.tx.min.size` +* dynamic: `false` + +`gds.cluster.tx.min.size` + +* Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +* defaultValue: `10000` +* validValues: an integer +* dynamic: `false` + + +== GDS Enterprise Edition + +`gds.enterprise.license_file` + +* Sets the location of the file that contains the Neo4j Graph Data Science library license key. +* defaultValue: `No Value` +* validValues: an absolute path +* dynamic: `false` + + +== GDS Export + +`gds.export.location` + +* Sets the export location for file based exports. +* defaultValue: `No Value` +* validValues: an absolute path +* dynamic: `false` + +`gds.model.store_location` label:enterprise-edition[Enterprise Edition] + +* Sets the location where persisted models are stored. +* defaultValue: `No Value` +* validValues: an absolute path +* dynamic: `false` + + +== Miscellaneous + +`gds.progress_tracking_enabled` + +* Enable progress logging tracking. +* defaultValue: `true` +* validValues: a boolean +* dynamic: `false` + +`gds.validate_using_max_memory_estimation` + +* Use maximum memory estimation in procedure memory guard. +* defaultValue: `false` +* validValues: a boolean +* dynamic: `false` diff --git a/doc/modules/ROOT/pages/production-deployment/index.adoc b/doc/modules/ROOT/pages/production-deployment/index.adoc index 2ebd38af2ac..f818030f53e 100644 --- a/doc/modules/ROOT/pages/production-deployment/index.adoc +++ b/doc/modules/ROOT/pages/production-deployment/index.adoc @@ -9,4 +9,5 @@ This chapter is divided into the following sections: * xref:production-deployment/transaction-handling.adoc[Transaction Handling] * xref:production-deployment/composite.adoc[Using GDS and Composite databases] * xref:production-deployment/neo4j-cluster.adoc[GDS with Neo4j cluster] +* xref:production-deployment/configuration-settings.adoc[GDS Configuration Settings] * xref:production-deployment/feature-toggles.adoc[GDS Feature Toggles] From 80016c17c0e8e73e18af266bf6ff13e34c9acfca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 9 Feb 2023 13:33:12 +0100 Subject: [PATCH 178/400] Bump Aura build number after 2.3.1+11 release --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 94f9fab04cd..50b45e7cfd5 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.1' - gdsAuraVersion = '11' + gdsAuraVersion = '12' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From c449600cd0ebeed62d965aa08f0c15c118ee68b3 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 8 Feb 2023 08:04:27 +0100 Subject: [PATCH 179/400] Never fail when no localMoves are ever done Co-authored-by: Veselin Nikolov --- .../java/org/neo4j/gds/leiden/Leiden.java | 22 ++++++++------- .../java/org/neo4j/gds/leiden/LeidenTest.java | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java index 0244ea21170..36204a4eb65 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java @@ -133,9 +133,10 @@ public LeidenResult compute() { gamma, concurrency ); - var communitiesCount = localMovePhase.run(); - boolean localPhaseConverged = communitiesCount == workingGraph.nodeCount() || localMovePhase.swaps == 0; + localMovePhase.run(); + //if you do swaps, no convergence + boolean localPhaseConverged = localMovePhase.swaps == 0; progressTracker.endSubTask("Local Move"); progressTracker.beginSubTask("Modularity Computation"); @@ -244,11 +245,11 @@ public LeidenResult compute() { @NotNull private LeidenResult getLeidenResult(boolean didConverge, int iteration) { - boolean seedIsOptimal = didConverge && seedValues.isPresent() && iteration == 0; - if (seedIsOptimal) { + boolean stoppedAtFirstIteration = didConverge && iteration == 0; + if (stoppedAtFirstIteration) { var modularity = modularities[0]; return LeidenResult.of( - LeidenUtils.createSeedCommunities(rootGraph.nodeCount(), seedValues.orElse(null)), + LeidenUtils.createStartingCommunities(rootGraph.nodeCount(), seedValues.orElse(null)), 1, didConverge, null, @@ -266,8 +267,8 @@ private LeidenResult getLeidenResult(boolean didConverge, int iteration) { ); } } - - private boolean updateModularity( + + private void updateModularity( Graph workingGraph, HugeLongArray localMoveCommunities, HugeDoubleArray localMoveCommunityVolumes, @@ -276,8 +277,10 @@ private boolean updateModularity( boolean localPhaseConverged, int iteration ) { - boolean seedIsOptimal = localPhaseConverged && seedValues.isPresent() && iteration == 0; - boolean shouldCalculateModularity = !localPhaseConverged || seedIsOptimal; + //will calculate modularity only if: + // - the local phase has not converged (i.e., no swaps done) + //- or we terminate in the first iteration (i.e., given seeding is optimal, graph is empty etc) + boolean shouldCalculateModularity = !localPhaseConverged || iteration == 0; if (shouldCalculateModularity) { modularities[iteration] = ModularityComputer.compute( @@ -291,7 +294,6 @@ private boolean updateModularity( progressTracker ); } - return seedIsOptimal; } private double initVolumes( diff --git a/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java b/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java index 193a1674ea1..7e040ee894e 100644 --- a/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java +++ b/algo/src/test/java/org/neo4j/gds/leiden/LeidenTest.java @@ -41,6 +41,7 @@ import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.extension.TestGraph; +import org.neo4j.gds.gdl.GdlFactory; import java.util.stream.Collectors; import java.util.stream.LongStream; @@ -154,6 +155,8 @@ void shouldWorkWithBestSeed() { community -> assertThat(community).containsExactlyInAnyOrder("a1", "a5", "a6", "a7") ); assertThat(communitiesMap.keySet()).containsExactly(4000L, 5000L); + + assertThat(leidenResult.modularity()).isGreaterThan(0); } @Test @@ -364,4 +367,29 @@ void shouldLogProgress() { "Leiden :: Finished" ); } + + @Test + void shouldWorkIfStopAtFirstIteration() { + + var query = " CREATE\n" + + " (c:C {id: 0}) \n" + + " , (d:D {id: 1}) "; + var empty = GdlFactory.of(query).build().getUnion(); + var result = new Leiden( + empty, + 5, + 1, + 0.01, + false, + 42, + null, + 0.1, + 1, + ProgressTracker.NULL_TRACKER + ).compute(); + var communities = result.communities(); + assertThat(communities.toArray()).isEqualTo(new long[]{0, 1}); + + + } } From 99b19c2f88ed1a75c22289c230ae3fc757e04ee8 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 9 Feb 2023 14:39:35 +0100 Subject: [PATCH 180/400] Correctly assign community volumes for weighted graphs Co-authored-by: Veselin Nikolov --- .../java/org/neo4j/gds/leiden/Leiden.java | 8 +- .../org/neo4j/gds/leiden/LocalMovePhase.java | 12 +-- .../org/neo4j/gds/leiden/LocalMoveTask.java | 7 +- .../gds/leiden/LeidenWeightedCliqueTest.java | 90 +++++++++++++++++++ 4 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 algo/src/test/java/org/neo4j/gds/leiden/LeidenWeightedCliqueTest.java diff --git a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java index 36204a4eb65..784ce0cbf41 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/Leiden.java @@ -277,9 +277,9 @@ private void updateModularity( boolean localPhaseConverged, int iteration ) { - //will calculate modularity only if: - // - the local phase has not converged (i.e., no swaps done) - //- or we terminate in the first iteration (i.e., given seeding is optimal, graph is empty etc) + // Will calculate modularity only if: + // - the local phase has not converged (i.e., no swaps done) + // - or we terminate in the first iteration (i.e., given seeding is optimal, graph is empty, etc) boolean shouldCalculateModularity = !localPhaseConverged || iteration == 0; if (shouldCalculateModularity) { @@ -329,7 +329,7 @@ private double initVolumes( rootGraph.forEachNode(nodeId -> { long communityId = initialCommunities.get(nodeId); progressTracker.logProgress(); - communityVolumes.addTo(communityId, rootGraph.degree(nodeId)); + communityVolumes.addTo(communityId, nodeVolumes.get(nodeId)); return true; }); progressTracker.endSubTask("Initialization"); diff --git a/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java b/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java index ad2d577bbb8..eac807a227c 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.leiden; -import org.apache.commons.lang3.mutable.MutableDouble; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.concurrency.RunWithConcurrency; import org.neo4j.gds.core.utils.mem.MemoryEstimation; @@ -103,10 +102,9 @@ private LocalMovePhase( } /** - * * @return The new community count. */ - public long run() { + public void run() { HugeAtomicDoubleArray atomicCommunityVolumes = HugeAtomicDoubleArray.newArray(graph.nodeCount()); graph.forEachNode(v -> { atomicCommunityVolumes.set(v, communityVolumes.get(v)); @@ -147,15 +145,11 @@ public long run() { swaps += task.swaps; } - MutableDouble aliveCommunities = new MutableDouble(graph.nodeCount()); graph.forEachNode(v -> { communityVolumes.set(v, atomicCommunityVolumes.get(v)); - if (Double.compare(communityVolumes.get(v), 0.0) == 0) { - aliveCommunities.decrement(); - } return true; }); - return aliveCommunities.longValue(); + } - + } diff --git a/algo/src/main/java/org/neo4j/gds/leiden/LocalMoveTask.java b/algo/src/main/java/org/neo4j/gds/leiden/LocalMoveTask.java index 0b6f42624cd..58f37752ab0 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/LocalMoveTask.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/LocalMoveTask.java @@ -129,8 +129,11 @@ private void moveNodeToNewCommunity( long oldCommunityId = currentCommunities.get(nodeId); currentCommunities.set(nodeId, newCommunityId); communityVolumes.getAndAdd(newCommunityId, currentNodeVolume); - communityVolumes.getAndAdd(oldCommunityId, -currentNodeVolume); - + //do a atomic update to never go into negatives etc + communityVolumes.update(oldCommunityId, (oldValue) -> { + var diff = oldValue - currentNodeVolume; + return Math.max(diff, 0.0); + }); swaps++; } diff --git a/algo/src/test/java/org/neo4j/gds/leiden/LeidenWeightedCliqueTest.java b/algo/src/test/java/org/neo4j/gds/leiden/LeidenWeightedCliqueTest.java new file mode 100644 index 00000000000..96d93d6be1d --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/leiden/LeidenWeightedCliqueTest.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.leiden; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.Orientation; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.IdFunction; +import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.extension.TestGraph; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.core.ProcedureConstants.TOLERANCE_DEFAULT; + +@GdlExtension +class LeidenWeightedCliqueTest { + + @GdlGraph(orientation = Orientation.UNDIRECTED) + private static final String DB_CYPHER = + "CREATE" + + "(a : Node), "+ + "(b : Node), "+ + "(c : Node), "+ + "(d : Node), "+ + "(e : Node), "+ + + "(a)-[:R {w: 0.1}]->(b),"+ + "(a)-[:R {w: 0.1}]->(c),"+ + "(a)-[:R {w: 0.1}]->(d),"+ + "(a)-[:R {w: 0.1}]->(e),"+ + + "(b)-[:R {w: 0.1}]->(c),"+ + "(b)-[:R {w: 0.1}]->(d),"+ + "(b)-[:R {w: 0.1}]->(e),"+ + + "(c)-[:R {w: 0.1}]->(d),"+ + "(c)-[:R {w: 0.1}]->(e),"+ + + "(d)-[:R {w: 0.1}]->(e)"; + + @Inject + private TestGraph graph; + + @Inject + private IdFunction idFunction; + + @Test + void weightedLeiden() { + var maxLevels = 10; + + Leiden leiden = new Leiden( + graph, + maxLevels, + 1.0, + 0.01, + true, + 19L, + null, + TOLERANCE_DEFAULT, + 4, + ProgressTracker.NULL_TRACKER + + ); + var leidenResult = leiden.compute(); + assertThat(Arrays.stream(leidenResult.communities().toArray()).distinct().count()).isEqualTo(1); + + } + +} From 0f2ebed6f1d10fa8e56bc4223b24d719be55be59 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 9 Feb 2023 14:40:22 +0100 Subject: [PATCH 181/400] Remove an array that is not used anymore Co-authored-by: Veselin Nikolov --- algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java b/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java index eac807a227c..9f7a25afe5b 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/LocalMovePhase.java @@ -71,9 +71,7 @@ static LocalMovePhase create( double gamma, int concurrency ) { - var encounteredCommunitiesWeights = HugeDoubleArray.newArray(graph.nodeCount()); - encounteredCommunitiesWeights.setAll(c -> -1L); - + return new LocalMovePhase( graph, seedCommunities, From c204aed89f1220fadeec6cc7713b03822602438e Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 1 Feb 2023 14:53:56 +0000 Subject: [PATCH 182/400] Allow to switch off progress tracking per procedure --- .../java/org/neo4j/gds/AlgorithmFactory.java | 23 +++--- .../nodesim/NodeSimilarityTest.java | 19 +++++ .../java/org/neo4j/gds/config/BaseConfig.java | 8 ++ .../neo4j/gds/core/loading/CypherFactory.java | 30 ++++--- .../neo4j/gds/core/loading/NativeFactory.java | 80 ++++++++++--------- .../neo4j/gds/test/ProgressTrackingTest.java | 78 ++++++++++++++++++ 6 files changed, 178 insertions(+), 60 deletions(-) create mode 100644 proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java diff --git a/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java b/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java index 525e868a618..9f19f603aff 100644 --- a/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java +++ b/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java @@ -55,15 +55,20 @@ default ALGO build( TaskRegistryFactory taskRegistryFactory, UserLogRegistryFactory userLogRegistryFactory ) { - var progressTask = progressTask(graphOrGraphStore, configuration); - var progressTracker = new TaskProgressTracker( - progressTask, - log, - configuration.concurrency(), - configuration.jobId(), - taskRegistryFactory, - userLogRegistryFactory - ); + ProgressTracker progressTracker; + if (configuration.logProgress()) { + var progressTask = progressTask(graphOrGraphStore, configuration); + progressTracker = new TaskProgressTracker( + progressTask, + log, + configuration.concurrency(), + configuration.jobId(), + taskRegistryFactory, + userLogRegistryFactory + ); + } else { + progressTracker = ProgressTracker.NULL_TRACKER; + } return build(graphOrGraphStore, configuration, progressTracker); } diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index f3c965233aa..b3b1bf6a7c0 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -799,6 +799,25 @@ void shouldLogMessages(int topK, int concurrency) { ); } + @ParameterizedTest(name = "topK = {0}, concurrency = {1}") + @MethodSource("topKAndConcurrencies") + void shouldNotLogMessagesWhenLoggingIsDisabled(int topK, int concurrency) { + var graph = naturalGraph; + var config = configBuilder().topN(100).topK(topK).concurrency(concurrency).logProgress(false).build(); + + var progressLog = Neo4jProxy.testLog(); + var nodeSimilarity = new NodeSimilarityFactory<>().build( + graph, + config, + progressLog, + EmptyTaskRegistryFactory.INSTANCE + ); + + nodeSimilarity.compute(); + + assertThat(progressLog.getMessages(INFO)).isEmpty(); + } + @ParameterizedTest(name = "concurrency = {0}") @ValueSource(ints = {1,2}) void shouldLogProgress(int concurrency) { diff --git a/config-api/src/main/java/org/neo4j/gds/config/BaseConfig.java b/config-api/src/main/java/org/neo4j/gds/config/BaseConfig.java index f6124f16b88..1a00b715862 100644 --- a/config-api/src/main/java/org/neo4j/gds/config/BaseConfig.java +++ b/config-api/src/main/java/org/neo4j/gds/config/BaseConfig.java @@ -32,6 +32,7 @@ public interface BaseConfig extends ToMapConvertible { String SUDO_KEY = "sudo"; + String LOG_PROGRESS_KEY = "logProgress"; @Value.Parameter(false) @Configuration.Key("username") @@ -45,6 +46,13 @@ default boolean sudo() { return false; } + @Value.Default + @Value.Parameter(false) + @Configuration.Key(LOG_PROGRESS_KEY) + default boolean logProgress() { + return true; + } + @Configuration.CollectKeys @Value.Auxiliary @Value.Default diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index d7c81c73cc7..050208fa374 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -160,19 +160,23 @@ public CSRGraphStore build() { @Override protected ProgressTracker initProgressTracker() { - var task = Tasks.task( - "Loading", - Tasks.leaf("Nodes"), - Tasks.leaf("Relationships", dimensions.relCountUpperBound()) - ); - return new TaskProgressTracker( - task, - loadingContext.log(), - graphProjectConfig.readConcurrency(), - graphProjectConfig.jobId(), - loadingContext.taskRegistryFactory(), - EmptyUserLogRegistryFactory.INSTANCE - ); + if (graphProjectConfig.logProgress()) { + var task = Tasks.task( + "Loading", + Tasks.leaf("Nodes"), + Tasks.leaf("Relationships", dimensions.relCountUpperBound()) + ); + return new TaskProgressTracker( + task, + loadingContext.log(), + graphProjectConfig.readConcurrency(), + graphProjectConfig.jobId(), + loadingContext.taskRegistryFactory(), + EmptyUserLogRegistryFactory.INSTANCE + ); + } + + return ProgressTracker.NULL_TRACKER; } private String nodeQuery() { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java index 4d0cd5d7a2a..01d261bb013 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java @@ -251,47 +251,51 @@ private static void relationshipEstimationAfterLoading( @Override protected ProgressTracker initProgressTracker() { - long relationshipCount = graphProjectConfig - .relationshipProjections() - .projections() - .entrySet() - .stream() - .map(entry -> { - long relCount = entry.getKey().name.equals("*") - ? dimensions.relationshipCounts().values().stream().reduce(Long::sum).orElse(0L) - : dimensions.relationshipCounts().getOrDefault(entry.getKey(), 0L); - - return entry.getValue().orientation() == Orientation.UNDIRECTED - ? relCount * 2 - : relCount; - }).mapToLong(Long::longValue).sum(); - - var properties = IndexPropertyMappings.prepareProperties( - graphProjectConfig, - dimensions, - loadingContext.transactionContext() - ); + if (graphProjectConfig.logProgress()) { + long relationshipCount = graphProjectConfig + .relationshipProjections() + .projections() + .entrySet() + .stream() + .map(entry -> { + long relCount = entry.getKey().name.equals("*") + ? dimensions.relationshipCounts().values().stream().reduce(Long::sum).orElse(0L) + : dimensions.relationshipCounts().getOrDefault(entry.getKey(), 0L); + + return entry.getValue().orientation() == Orientation.UNDIRECTED + ? relCount * 2 + : relCount; + }).mapToLong(Long::longValue).sum(); + + var properties = IndexPropertyMappings.prepareProperties( + graphProjectConfig, + dimensions, + loadingContext.transactionContext() + ); - List nodeTasks = properties.indexedProperties().isEmpty() - ? List.of(Tasks.leaf("Store Scan", dimensions.nodeCount())) - : List.of( - Tasks.leaf("Store Scan", dimensions.nodeCount()), - Tasks.leaf("Property Index Scan", properties.indexedProperties().size() * dimensions.nodeCount()) + List nodeTasks = properties.indexedProperties().isEmpty() + ? List.of(Tasks.leaf("Store Scan", dimensions.nodeCount())) + : List.of( + Tasks.leaf("Store Scan", dimensions.nodeCount()), + Tasks.leaf("Property Index Scan", properties.indexedProperties().size() * dimensions.nodeCount()) + ); + + var task = Tasks.task( + "Loading", + Tasks.task("Nodes", nodeTasks), + Tasks.task("Relationships", Tasks.leaf("Store Scan", relationshipCount)) ); + return new TaskProgressTracker( + task, + loadingContext.log(), + graphProjectConfig.readConcurrency(), + graphProjectConfig.jobId(), + loadingContext.taskRegistryFactory(), + EmptyUserLogRegistryFactory.INSTANCE + ); + } - var task = Tasks.task( - "Loading", - Tasks.task("Nodes", nodeTasks), - Tasks.task("Relationships", Tasks.leaf("Store Scan", relationshipCount)) - ); - return new TaskProgressTracker( - task, - loadingContext.log(), - graphProjectConfig.readConcurrency(), - graphProjectConfig.jobId(), - loadingContext.taskRegistryFactory(), - EmptyUserLogRegistryFactory.INSTANCE - ); + return ProgressTracker.NULL_TRACKER; } @Override diff --git a/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java b/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java new file mode 100644 index 00000000000..e2d7c550340 --- /dev/null +++ b/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.test; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.assertj.Extractors.removingThreadId; +import static org.neo4j.gds.assertj.Extractors.replaceTimings; + +@GdlExtension +class ProgressTrackingTest { + + @GdlGraph + static String GDL = + "CREATE " + + " ()-[:REL]->()," + + " ()-[:REL2]->(),"; + + @Inject + Graph graph; + + @Test + void shouldLogProgress() { + var factory = new TestAlgorithmFactory<>(); + var testConfig = TestConfigImpl.builder().logProgress(true).build(); + var log = Neo4jProxy.testLog(); + + factory.build(graph, testConfig, log, TaskRegistryFactory.empty()).compute(); + + assertThat(log.getMessages(TestLog.INFO)) + .extracting(removingThreadId()) + .extracting(replaceTimings()) + .containsExactly( + "TestAlgorithm :: Start", + "TestAlgorithm 100%", + "TestAlgorithm :: Finished" + ); + + } + + @Test + void shouldNotLogProgress() { + var factory = new TestAlgorithmFactory<>(); + var testConfig = TestConfigImpl.builder().logProgress(false).build(); + var log = Neo4jProxy.testLog(); + + factory.build(graph, testConfig, log, TaskRegistryFactory.empty()).compute(); + + assertThat(log.getMessages(TestLog.INFO)) + .as("When `logProgress` is set to `false` there should be no log messages") + .isEmpty(); + } +} From 23c866f62769087fc7361c43fef98785e61055df Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 7 Feb 2023 03:03:40 +0000 Subject: [PATCH 183/400] Add ProgressTracker that doesn't log inner tasks progress Co-authored-by: Ioannis Panagiotas --- .../java/org/neo4j/gds/AlgorithmFactory.java | 32 +++++++- .../org/neo4j/gds/test/TestAlgorithm.java | 3 +- .../nodesim/NodeSimilarityTest.java | 14 +++- .../neo4j/gds/core/loading/CypherFactory.java | 19 +++-- .../neo4j/gds/core/loading/NativeFactory.java | 75 +++++++++++-------- .../progress/tasks/TaskProgressTracker.java | 6 +- .../tasks/TaskTreeProgressTracker.java | 54 +++++++++++++ .../neo4j/gds/catalog/GraphDropProcTest.java | 3 +- .../neo4j/gds/catalog/GraphListProcTest.java | 14 ++-- .../graphsage/GraphSageIntegrationTest.java | 2 +- .../LinkPredictionPipelineTrainProcTest.java | 4 +- ...deClassificationPipelineTrainProcTest.java | 2 +- .../NodeRegressionPipelineTrainProcTest.java | 2 +- .../neo4j/gds/test/ProgressTrackingTest.java | 29 ++++++- 14 files changed, 198 insertions(+), 61 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java diff --git a/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java b/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java index 9f19f603aff..9b1262ce702 100644 --- a/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java +++ b/algo-common/src/main/java/org/neo4j/gds/AlgorithmFactory.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds; +import org.jetbrains.annotations.NotNull; import org.neo4j.gds.config.AlgoBaseConfig; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.utils.mem.MemoryEstimation; @@ -26,6 +27,7 @@ import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.TaskTreeProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory; import org.neo4j.gds.core.utils.warnings.UserLogRegistryFactory; @@ -54,10 +56,27 @@ default ALGO build( Log log, TaskRegistryFactory taskRegistryFactory, UserLogRegistryFactory userLogRegistryFactory + ) { + var progressTracker = createProgressTracker( + configuration, + log, + taskRegistryFactory, + userLogRegistryFactory, + progressTask(graphOrGraphStore, configuration) + ); + return build(graphOrGraphStore, configuration, progressTracker); + } + + @NotNull + private ProgressTracker createProgressTracker( + CONFIG configuration, + Log log, + TaskRegistryFactory taskRegistryFactory, + UserLogRegistryFactory userLogRegistryFactory, + Task progressTask ) { ProgressTracker progressTracker; if (configuration.logProgress()) { - var progressTask = progressTask(graphOrGraphStore, configuration); progressTracker = new TaskProgressTracker( progressTask, log, @@ -67,9 +86,16 @@ default ALGO build( userLogRegistryFactory ); } else { - progressTracker = ProgressTracker.NULL_TRACKER; + progressTracker = new TaskTreeProgressTracker( + progressTask, + log, + configuration.concurrency(), + configuration.jobId(), + taskRegistryFactory, + userLogRegistryFactory + ); } - return build(graphOrGraphStore, configuration, progressTracker); + return progressTracker; } ALGO build( diff --git a/algo-test/src/main/java/org/neo4j/gds/test/TestAlgorithm.java b/algo-test/src/main/java/org/neo4j/gds/test/TestAlgorithm.java index 64250a7bb8c..f224286a0c1 100644 --- a/algo-test/src/main/java/org/neo4j/gds/test/TestAlgorithm.java +++ b/algo-test/src/main/java/org/neo4j/gds/test/TestAlgorithm.java @@ -41,11 +41,12 @@ public TestAlgorithm( @Override public TestAlgorithm compute() { - progressTracker.beginSubTask(); + progressTracker.beginSubTask(100); if (throwInCompute) { throw new IllegalStateException("boo"); } + progressTracker.logProgress(50); relationshipCount = graph.relationshipCount(); progressTracker.endSubTask(); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index b3b1bf6a7c0..03f12709f2e 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -815,7 +815,19 @@ void shouldNotLogMessagesWhenLoggingIsDisabled(int topK, int concurrency) { nodeSimilarity.compute(); - assertThat(progressLog.getMessages(INFO)).isEmpty(); + assertThat(progressLog.getMessages(INFO)) + .as("When progress logging is disabled we only log `start`, `100%` and `finished`.") + .extracting(removingThreadId()) + .containsExactly( + "NodeSimilarity :: Start", + "NodeSimilarity :: prepare :: Start", + "NodeSimilarity :: prepare 100%", + "NodeSimilarity :: prepare :: Finished", + "NodeSimilarity :: compare node pairs :: Start", + "NodeSimilarity :: compare node pairs 100%", + "NodeSimilarity :: compare node pairs :: Finished", + "NodeSimilarity :: Finished" + ); } @ParameterizedTest(name = "concurrency = {0}") diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index 050208fa374..a91e657fc2d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -160,12 +160,12 @@ public CSRGraphStore build() { @Override protected ProgressTracker initProgressTracker() { + var task = Tasks.task( + "Loading", + Tasks.leaf("Nodes"), + Tasks.leaf("Relationships", dimensions.relCountUpperBound()) + ); if (graphProjectConfig.logProgress()) { - var task = Tasks.task( - "Loading", - Tasks.leaf("Nodes"), - Tasks.leaf("Relationships", dimensions.relCountUpperBound()) - ); return new TaskProgressTracker( task, loadingContext.log(), @@ -176,7 +176,14 @@ protected ProgressTracker initProgressTracker() { ); } - return ProgressTracker.NULL_TRACKER; + return new TaskProgressTracker( + task, + loadingContext.log(), + graphProjectConfig.readConcurrency(), + graphProjectConfig.jobId(), + loadingContext.taskRegistryFactory(), + EmptyUserLogRegistryFactory.INSTANCE + ); } private String nodeQuery() { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java index 01d261bb013..b11c42c6921 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java @@ -43,6 +43,7 @@ import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.TaskTreeProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory; import org.neo4j.internal.id.IdGeneratorFactory; @@ -251,40 +252,41 @@ private static void relationshipEstimationAfterLoading( @Override protected ProgressTracker initProgressTracker() { - if (graphProjectConfig.logProgress()) { - long relationshipCount = graphProjectConfig - .relationshipProjections() - .projections() - .entrySet() - .stream() - .map(entry -> { - long relCount = entry.getKey().name.equals("*") - ? dimensions.relationshipCounts().values().stream().reduce(Long::sum).orElse(0L) - : dimensions.relationshipCounts().getOrDefault(entry.getKey(), 0L); - - return entry.getValue().orientation() == Orientation.UNDIRECTED - ? relCount * 2 - : relCount; - }).mapToLong(Long::longValue).sum(); - - var properties = IndexPropertyMappings.prepareProperties( - graphProjectConfig, - dimensions, - loadingContext.transactionContext() - ); + long relationshipCount = graphProjectConfig + .relationshipProjections() + .projections() + .entrySet() + .stream() + .map(entry -> { + long relCount = entry.getKey().name.equals("*") + ? dimensions.relationshipCounts().values().stream().reduce(Long::sum).orElse(0L) + : dimensions.relationshipCounts().getOrDefault(entry.getKey(), 0L); + + return entry.getValue().orientation() == Orientation.UNDIRECTED + ? relCount * 2 + : relCount; + }).mapToLong(Long::longValue).sum(); + + var properties = IndexPropertyMappings.prepareProperties( + graphProjectConfig, + dimensions, + loadingContext.transactionContext() + ); - List nodeTasks = properties.indexedProperties().isEmpty() - ? List.of(Tasks.leaf("Store Scan", dimensions.nodeCount())) - : List.of( - Tasks.leaf("Store Scan", dimensions.nodeCount()), - Tasks.leaf("Property Index Scan", properties.indexedProperties().size() * dimensions.nodeCount()) - ); - - var task = Tasks.task( - "Loading", - Tasks.task("Nodes", nodeTasks), - Tasks.task("Relationships", Tasks.leaf("Store Scan", relationshipCount)) + List nodeTasks = properties.indexedProperties().isEmpty() + ? List.of(Tasks.leaf("Store Scan", dimensions.nodeCount())) + : List.of( + Tasks.leaf("Store Scan", dimensions.nodeCount()), + Tasks.leaf("Property Index Scan", properties.indexedProperties().size() * dimensions.nodeCount()) ); + + var task = Tasks.task( + "Loading", + Tasks.task("Nodes", nodeTasks), + Tasks.task("Relationships", Tasks.leaf("Store Scan", relationshipCount)) + ); + + if (graphProjectConfig.logProgress()) { return new TaskProgressTracker( task, loadingContext.log(), @@ -295,7 +297,14 @@ protected ProgressTracker initProgressTracker() { ); } - return ProgressTracker.NULL_TRACKER; + return new TaskTreeProgressTracker( + task, + loadingContext.log(), + graphProjectConfig.readConcurrency(), + graphProjectConfig.jobId(), + loadingContext.taskRegistryFactory(), + EmptyUserLogRegistryFactory.INSTANCE + ); } @Override diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java index eb752cb77ea..a094acb31b5 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java @@ -51,7 +51,7 @@ public class TaskProgressTracker implements ProgressTracker { private long currentTotalSteps; private double progressLeftOvers; - private Runnable onError; + private final Runnable onError; public TaskProgressTracker(Task baseTask, Log log, int concurrency, TaskRegistryFactory taskRegistryFactory) { this(baseTask, log, concurrency, new JobId(), taskRegistryFactory, EmptyUserLogRegistryFactory.INSTANCE); @@ -253,9 +253,9 @@ public void endSubTaskWithFailure(String expectedTaskDescription) { } @TestOnly - public Task currentSubTask() { + Task currentSubTask() { requireCurrentTask(); - return currentTask.get(); + return currentTask.orElseThrow(); } @Nullable diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java new file mode 100644 index 00000000000..361057919f7 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.utils.progress.tasks; + +import org.neo4j.gds.core.utils.progress.JobId; +import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.core.utils.warnings.UserLogRegistryFactory; +import org.neo4j.logging.Log; + +public final class TaskTreeProgressTracker extends TaskProgressTracker { + + public TaskTreeProgressTracker( + Task baseTask, + Log log, + int concurrency, + JobId jobId, + TaskRegistryFactory taskRegistryFactory, + UserLogRegistryFactory userLogRegistryFactory + ) { + super(baseTask, log, concurrency, jobId, taskRegistryFactory, userLogRegistryFactory); + } + + @Override + public void logSteps(long steps) { + // NOOP + } + + @Override + public void logProgress(long value) { + // NOOP + } + + @Override + public void logProgress(long value, String messageTemplate) { + // NOOP + } +} diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropProcTest.java index 3a09f3d0acc..390b6f7d970 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropProcTest.java @@ -94,7 +94,7 @@ void dropGraphFromCatalog() { new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(9) + .hasSize(10) .containsEntry( "nodeProjection", map( "A", map( @@ -126,6 +126,7 @@ void dropGraphFromCatalog() { intAssertConsumer(readConcurrency -> readConcurrency.isEqualTo(4)) ) .hasEntrySatisfying("sudo", booleanAssertConsumer(AbstractBooleanAssert::isFalse)) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) .doesNotContainKeys( "username", GraphProjectConfig.NODE_COUNT_KEY, diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListProcTest.java index 3d181f9c95c..f6e2524cfaa 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphListProcTest.java @@ -104,7 +104,7 @@ void listASingleLabelRelationshipTypeProjection() { new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(9) + .hasSize(10) .containsEntry( "nodeProjection", map( "A", map( @@ -136,6 +136,7 @@ void listASingleLabelRelationshipTypeProjection() { intAssertConsumer(readConcurrency -> readConcurrency.isEqualTo(4)) ) .hasEntrySatisfying("sudo", booleanAssertConsumer(AbstractBooleanAssert::isFalse)) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) .doesNotContainKeys( GraphProjectConfig.NODE_COUNT_KEY, GraphProjectConfig.RELATIONSHIP_COUNT_KEY, @@ -217,7 +218,7 @@ void listGeneratedGraph() { "configuration", new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(11) + .hasSize(12) .containsEntry("nodeProjections", map( "10_Nodes", map( "label", "10_Nodes", @@ -250,6 +251,7 @@ void listGeneratedGraph() { ) .hasEntrySatisfying("allowSelfLoops", booleanAssertConsumer(AbstractBooleanAssert::isFalse)) .hasEntrySatisfying("sudo", booleanAssertConsumer(AbstractBooleanAssert::isFalse)) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) .hasEntrySatisfying( "relationshipDistribution", stringAssertConsumer(relationshipDistribution -> relationshipDistribution.isEqualTo( @@ -344,7 +346,7 @@ void listCypherProjection() { "configuration", new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(8) + .hasSize(9) .hasEntrySatisfying( "relationshipQuery", stringAssertConsumer(relationshipQuery -> relationshipQuery.isEqualTo( @@ -360,6 +362,7 @@ void listCypherProjection() { stringAssertConsumer(nodeQuery -> nodeQuery.isEqualTo(ALL_NODES_QUERY)) ) .hasEntrySatisfying("sudo", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)) .hasEntrySatisfying( "readConcurrency", intAssertConsumer(readConcurrency -> readConcurrency.isEqualTo(4)) @@ -417,11 +420,12 @@ void listCypherAggregation() { "configuration", new Condition<>(config -> { assertThat(config) .asInstanceOf(stringObjectMapAssertFactory()) - .hasSize(4) + .hasSize(5) .hasEntrySatisfying("creationTime", creationTimeAssertConsumer()) .hasEntrySatisfying("jobId", jobId -> assertThat(jobId).isNotNull()) .hasEntrySatisfying("undirectedRelationshipTypes", t -> assertThat(t).isEqualTo(List.of())) - .hasEntrySatisfying("inverseIndexedRelationshipTypes", t -> assertThat(t).isEqualTo(List.of())); + .hasEntrySatisfying("inverseIndexedRelationshipTypes", t -> assertThat(t).isEqualTo(List.of())) + .hasEntrySatisfying("logProgress", booleanAssertConsumer(AbstractBooleanAssert::isTrue)); return true; }, "Assert Cypher Aggregation `configuration` map"), diff --git a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageIntegrationTest.java b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageIntegrationTest.java index 05b432bd1e9..59d8108d528 100644 --- a/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageIntegrationTest.java +++ b/proc/embeddings/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageIntegrationTest.java @@ -113,7 +113,7 @@ private void dropModel() { ), "creationTime", isA(ZonedDateTime.class), "trainConfig", allOf( - aMapWithSize(19), + aMapWithSize(20), hasEntry("modelName", modelName), hasEntry("aggregator", "MEAN"), hasEntry("activationFunction", "SIGMOID") diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainProcTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainProcTest.java index 6ba65be0705..acc1538ddca 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainProcTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/linkmodels/pipeline/train/LinkPredictionPipelineTrainProcTest.java @@ -184,7 +184,7 @@ void trainAModel() { Matchers.hasKey("bestParameters") ), "trainMillis", greaterThan(-1L), - "configuration", aMapWithSize(12) + "configuration", aMapWithSize(13) )) ); @@ -257,7 +257,7 @@ void trainOnNodeLabelFilteredGraph() { Matchers.hasKey("bestParameters") ), "trainMillis", greaterThan(-1L), - "configuration", aMapWithSize(12) + "configuration", aMapWithSize(13) )) ); GraphStore graphStore = GraphStoreCatalog.get(getUsername(), DatabaseId.of(db), GRAPH_NAME).graphStore(); diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainProcTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainProcTest.java index 5b131e0fa4a..555f60ac994 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainProcTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineTrainProcTest.java @@ -268,7 +268,7 @@ void train() { "configuration", Matchers.allOf( Matchers.hasEntry("pipeline", PIPELINE_NAME), Matchers.hasEntry("modelName", MODEL_NAME), - aMapWithSize(11) + aMapWithSize(12) ), "modelSelectionStats", modelSelectionStatsCheck ) diff --git a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainProcTest.java b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainProcTest.java index ecf128ec9d2..5f4ddc623df 100644 --- a/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainProcTest.java +++ b/proc/machine-learning/src/test/java/org/neo4j/gds/ml/pipeline/node/regression/NodeRegressionPipelineTrainProcTest.java @@ -233,7 +233,7 @@ void train() { "configuration", Matchers.allOf( Matchers.hasEntry("pipeline", PIPELINE_NAME), Matchers.hasEntry("modelName", MODEL_NAME), - aMapWithSize(11) + aMapWithSize(12) ), "modelSelectionStats",modelSelectionStatsCheck ) diff --git a/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java b/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java index e2d7c550340..25d1ea4ec4e 100644 --- a/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java +++ b/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java @@ -23,12 +23,20 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.utils.progress.JobId; +import org.neo4j.gds.core.utils.progress.TaskRegistry; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; +import org.neo4j.gds.core.utils.progress.tasks.Task; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; import org.neo4j.gds.extension.Inject; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.neo4j.gds.assertj.Extractors.removingThreadId; import static org.neo4j.gds.assertj.Extractors.replaceTimings; @@ -57,6 +65,7 @@ void shouldLogProgress() { .extracting(replaceTimings()) .containsExactly( "TestAlgorithm :: Start", + "TestAlgorithm 50%", "TestAlgorithm 100%", "TestAlgorithm :: Finished" ); @@ -69,10 +78,24 @@ void shouldNotLogProgress() { var testConfig = TestConfigImpl.builder().logProgress(false).build(); var log = Neo4jProxy.testLog(); - factory.build(graph, testConfig, log, TaskRegistryFactory.empty()).compute(); + TaskRegistryFactory taskRegistryFactoryMock = mock(TaskRegistryFactory.class); + TaskRegistry taskRegistryMock = mock(TaskRegistry.class); + doReturn(taskRegistryMock).when(taskRegistryFactoryMock).newInstance(any(JobId.class)); + + factory.build(graph, testConfig, log, taskRegistryFactoryMock).compute(); assertThat(log.getMessages(TestLog.INFO)) - .as("When `logProgress` is set to `false` there should be no log messages") - .isEmpty(); + .as("When `logProgress` is set to `false` there should only be `start`, `100%` and `finished` log messages") + .extracting(removingThreadId()) + .extracting(replaceTimings()) + .containsExactly( + "TestAlgorithm :: Start", + "TestAlgorithm 100%", + "TestAlgorithm :: Finished" + ); + + // Now make sure that the tasks have been registered + verify(taskRegistryMock, times(1)).registerTask(any(Task.class)); + verify(taskRegistryMock, times(1)).unregisterTask(); } } From 58d80ecf6b85346042e59de71fe834b91bb7361d Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 10 Feb 2023 08:56:24 +0000 Subject: [PATCH 184/400] Don't log any percentage if logProgress is disabled Co-authored-by: Ioannis Panagiotas --- .../nodesim/NodeSimilarityTest.java | 4 +--- .../progress/tasks/TaskProgressLogger.java | 8 +++++++- .../progress/tasks/TaskProgressTracker.java | 18 +++++++++++++++-- .../tasks/TaskTreeProgressTracker.java | 20 ++++++++++++++++++- .../neo4j/gds/test/ProgressTrackingTest.java | 3 +-- 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index 03f12709f2e..ae9ab9fefe1 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -816,15 +816,13 @@ void shouldNotLogMessagesWhenLoggingIsDisabled(int topK, int concurrency) { nodeSimilarity.compute(); assertThat(progressLog.getMessages(INFO)) - .as("When progress logging is disabled we only log `start`, `100%` and `finished`.") + .as("When progress logging is disabled we only log `start` and `finished`.") .extracting(removingThreadId()) .containsExactly( "NodeSimilarity :: Start", "NodeSimilarity :: prepare :: Start", - "NodeSimilarity :: prepare 100%", "NodeSimilarity :: prepare :: Finished", "NodeSimilarity :: compare node pairs :: Start", - "NodeSimilarity :: compare node pairs 100%", "NodeSimilarity :: compare node pairs :: Finished", "NodeSimilarity :: Finished" ); diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressLogger.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressLogger.java index 6822ce40229..b08a92330a0 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressLogger.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressLogger.java @@ -29,12 +29,18 @@ class TaskProgressLogger extends BatchingProgressLogger { private final Task baseTask; - private final LoggingLeafTaskVisitor loggingLeafTaskVisitor; + private final TaskVisitor loggingLeafTaskVisitor; TaskProgressLogger(Log log, Task baseTask, int concurrency) { super(log, baseTask, concurrency); this.baseTask = baseTask; this.loggingLeafTaskVisitor = new LoggingLeafTaskVisitor(this); + + } + TaskProgressLogger(Log log, Task baseTask, int concurrency, TaskVisitor leafTaskVisitor) { + super(log, baseTask, concurrency); + this.baseTask = baseTask; + this.loggingLeafTaskVisitor = leafTaskVisitor; } void logBeginSubTask(Task task, Task parentTask) { diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java index a094acb31b5..cb2a7c43bac 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskProgressTracker.java @@ -58,12 +58,26 @@ public TaskProgressTracker(Task baseTask, Log log, int concurrency, TaskRegistry } public TaskProgressTracker( - Task baseTask, Log log, int concurrency, JobId jobId, TaskRegistryFactory taskRegistryFactory, + Task baseTask, + Log log, + int concurrency, + JobId jobId, + TaskRegistryFactory taskRegistryFactory, + UserLogRegistryFactory userLogRegistryFactory + ) { + this(baseTask, jobId, taskRegistryFactory, new TaskProgressLogger(log, baseTask, concurrency), userLogRegistryFactory); + } + + protected TaskProgressTracker( + Task baseTask, + JobId jobId, + TaskRegistryFactory taskRegistryFactory, + TaskProgressLogger taskProgressLogger, UserLogRegistryFactory userLogRegistryFactory ) { this.baseTask = baseTask; this.taskRegistry = taskRegistryFactory.newInstance(jobId); - this.taskProgressLogger = new TaskProgressLogger(log, baseTask, concurrency); + this.taskProgressLogger = taskProgressLogger; this.currentTask = Optional.empty(); this.currentTotalSteps = UNKNOWN_STEPS; this.progressLeftOvers = 0; diff --git a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java index 361057919f7..38c7d57f442 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/progress/tasks/TaskTreeProgressTracker.java @@ -34,7 +34,18 @@ public TaskTreeProgressTracker( TaskRegistryFactory taskRegistryFactory, UserLogRegistryFactory userLogRegistryFactory ) { - super(baseTask, log, concurrency, jobId, taskRegistryFactory, userLogRegistryFactory); + super( + baseTask, + jobId, + taskRegistryFactory, + new TaskProgressLogger( + log, + baseTask, + concurrency, + new PassThroughTaskVisitor() + ), + userLogRegistryFactory + ); } @Override @@ -51,4 +62,11 @@ public void logProgress(long value) { public void logProgress(long value, String messageTemplate) { // NOOP } + + private static class PassThroughTaskVisitor implements TaskVisitor { + @Override + public void visitLeafTask(LeafTask leafTask) { + // NOOP --> just pass through + } + } } diff --git a/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java b/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java index 25d1ea4ec4e..ab69529e2bf 100644 --- a/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java +++ b/proc/test/src/test/java/org/neo4j/gds/test/ProgressTrackingTest.java @@ -85,12 +85,11 @@ void shouldNotLogProgress() { factory.build(graph, testConfig, log, taskRegistryFactoryMock).compute(); assertThat(log.getMessages(TestLog.INFO)) - .as("When `logProgress` is set to `false` there should only be `start`, `100%` and `finished` log messages") + .as("When `logProgress` is set to `false` there should only be `start` and `finished` log messages") .extracting(removingThreadId()) .extracting(replaceTimings()) .containsExactly( "TestAlgorithm :: Start", - "TestAlgorithm 100%", "TestAlgorithm :: Finished" ); From 07560667d54f14b88262ff7923676bdb580ed42e Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 10 Feb 2023 10:17:17 +0000 Subject: [PATCH 185/400] Document `logProgress` configuration parameter Co-authored-by: Ioannis Panagiotas --- doc/modules/ROOT/pages/common-usage/running-algos.adoc | 4 ++++ .../common-configuration-jobid-concurrency-entries.adoc | 1 + .../algorithms/common-configuration/common-configuration.adoc | 1 + 3 files changed, 6 insertions(+) diff --git a/doc/modules/ROOT/pages/common-usage/running-algos.adoc b/doc/modules/ROOT/pages/common-usage/running-algos.adoc index 456e4040e0c..4692cc97412 100644 --- a/doc/modules/ROOT/pages/common-usage/running-algos.adoc +++ b/doc/modules/ROOT/pages/common-usage/running-algos.adoc @@ -130,3 +130,7 @@ The Default is `concurrency` [[common-configuration-jobid]] jobId - String:: An id for the job to be started can be provided in order for it to be more easily tracked with eg. GDS's xref:common-usage/logging.adoc[logging capabilities]. + +[[common-configuration-logProgress]] +logProgress - Boolean:: +Configuration parameter that allows to turn `off/on` percentage logging while running procedure. It is `on` by default diff --git a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc index 3beba6c5e4c..dcbdb16cab5 100644 --- a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc +++ b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc @@ -1,2 +1,3 @@ | xref:common-usage/running-algos.adoc#common-configuration-concurrency[concurrency] | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. | xref:common-usage/running-algos.adoc#common-configuration-jobid[jobId] | String | Generated internally | yes | An ID that can be provided to more easily track the algorithm's progress. +| xref:common-usage/running-algos.adoc#common-configuration-logProgress[logProgress] | Boolean | true | yes | If disabled the progress percentage will not be logged. diff --git a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration.adoc b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration.adoc index 55f9fbdac01..141226463eb 100644 --- a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration.adoc @@ -4,4 +4,5 @@ | Name | Type | Default | Optional | Description | xref:common-usage/running-algos.adoc#common-configuration-concurrency[concurrency] | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. Also provides the default value for 'readConcurrency' and 'writeConcurrency'. | xref:common-usage/running-algos.adoc#common-configuration-write-concurrency[writeConcurrency] | Integer | value of 'concurrency' | yes | The number of concurrent threads used for writing the result (applicable in WRITE mode). +| xref:common-usage/running-algos.adoc#common-configuration-logProgress[logProgress] | Boolean | true | yes | If disabled the progress percentage will not be logged. |=== From 3ebfaff603344e7aa3c580f56588be022da88874 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 10 Feb 2023 12:11:16 +0100 Subject: [PATCH 186/400] Fix incorrect workload calculation for filtered node similarity Co-authored-by: Veselin Nikolov --- .../similarity/nodesim/NodeSimilarity.java | 10 +++- .../FilteredNodeSimilarityTest.java | 59 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java index dad85ebf44a..5ec743f2a9f 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java @@ -437,8 +437,14 @@ private LongStream checkProgress(LongStream stream) { } private long calculateWorkload() { - long workload = nodesToCompare * nodesToCompare; - if (concurrency == 1) { + //for each source node, examine all their target nodes + //if no filter then sourceNodes == targetNodes + long workload = sourceNodes.cardinality() * targetNodes.cardinality(); + + //when on concurrency of 1 on not-filtered similarity, we only compare nodeId with greater indexed nodes + // so work is halved. This does not hold for filtered similarity, since the targetNodes might be lesser indexed. + boolean isNotFiltered = sourceNodes.equals(NodeFilter.noOp) && targetNodeFilter.equals(NodeFilter.noOp); + if (concurrency == 1 && isNotFiltered) { workload = workload / 2; } return workload; diff --git a/algo/src/test/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityTest.java index 83d5eb62b87..093f5b7e549 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityTest.java @@ -20,6 +20,12 @@ package org.neo4j.gds.similarity.filterednodesim; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.TestProgressTracker; +import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; @@ -30,6 +36,8 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.neo4j.gds.assertj.Extractors.removingThreadId; +import static org.neo4j.gds.compat.TestLog.INFO; @GdlExtension class FilteredNodeSimilarityTest { @@ -154,4 +162,55 @@ void shouldSurviveIoannisFurtherObjections() { nodeSimilarity.release(); } + + @ParameterizedTest + @ValueSource(ints = {1, 2}) + void shouldLogProgressAccurately(int concurrency) { + var sourceNodeFilter = List.of(2L, 3L); + + var config = ImmutableFilteredNodeSimilarityStreamConfig.builder() + .sourceNodeFilter(NodeFilterSpecFactory.create(sourceNodeFilter)) + .concurrency(concurrency) + .topK(0) + .topN(10) + .build(); + var progressTask = new FilteredNodeSimilarityFactory<>().progressTask(graph, config); + TestLog log = Neo4jProxy.testLog(); + var progressTracker = new TestProgressTracker( + progressTask, + log, + concurrency, + EmptyTaskRegistryFactory.INSTANCE + ); + + + new FilteredNodeSimilarityFactory<>().build( + graph, + config, + progressTracker + ).compute(); + + + assertThat(log.getMessages(INFO)) + .extracting(removingThreadId()) + .containsExactly( + "FilteredNodeSimilarity :: Start", + "FilteredNodeSimilarity :: prepare :: Start", + "FilteredNodeSimilarity :: prepare 33%", + "FilteredNodeSimilarity :: prepare 55%", + "FilteredNodeSimilarity :: prepare 66%", + "FilteredNodeSimilarity :: prepare 100%", + "FilteredNodeSimilarity :: prepare :: Finished", + "FilteredNodeSimilarity :: compare node pairs :: Start", + "FilteredNodeSimilarity :: compare node pairs 12%", + "FilteredNodeSimilarity :: compare node pairs 25%", + "FilteredNodeSimilarity :: compare node pairs 37%", + "FilteredNodeSimilarity :: compare node pairs 50%", + "FilteredNodeSimilarity :: compare node pairs 62%", + "FilteredNodeSimilarity :: compare node pairs 75%", + "FilteredNodeSimilarity :: compare node pairs 100%", + "FilteredNodeSimilarity :: compare node pairs :: Finished", + "FilteredNodeSimilarity :: Finished" + ); + } } From 2ea9418a549a5b82682c3e6f56f947e51ff0b91c Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 13 Feb 2023 13:23:00 +0100 Subject: [PATCH 187/400] Inline field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java index 5ec743f2a9f..6611edf810f 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java @@ -57,7 +57,6 @@ public class NodeSimilarity extends Algorithm { private final MetricSimilarityComputer similarityComputer; private HugeObjectArray vectors; private HugeObjectArray weights; - private long nodesToCompare; private final boolean weighted; @@ -188,7 +187,7 @@ public SimilarityGraphResult computeToGraph() { executorService ).build(similarities); } - return new SimilarityGraphResult(similarityGraph, nodesToCompare, isTopKGraph); + return new SimilarityGraphResult(similarityGraph, sourceNodes.cardinality(), isTopKGraph); } private void prepare() { @@ -230,7 +229,6 @@ private void prepare() { progressTracker.logProgress(graph.degree(node)); return null; }); - nodesToCompare = sourceNodes.cardinality(); progressTracker.endSubTask(); } From 1a7bcfd312cc657b742b09bcdbf2113e4af9bb2b Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 13 Feb 2023 14:36:22 +0100 Subject: [PATCH 188/400] Correct filter call --- .../java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java index 6611edf810f..af069394e5d 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java @@ -441,7 +441,7 @@ private long calculateWorkload() { //when on concurrency of 1 on not-filtered similarity, we only compare nodeId with greater indexed nodes // so work is halved. This does not hold for filtered similarity, since the targetNodes might be lesser indexed. - boolean isNotFiltered = sourceNodes.equals(NodeFilter.noOp) && targetNodeFilter.equals(NodeFilter.noOp); + boolean isNotFiltered = sourceNodeFilter.equals(NodeFilter.noOp) && targetNodeFilter.equals(NodeFilter.noOp); if (concurrency == 1 && isNotFiltered) { workload = workload / 2; } From e9d8b9cefba67908a962e890aeda4673e5d1975a Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Feb 2023 11:22:56 +0100 Subject: [PATCH 189/400] Find and solve the issue Co-authored-by: Veselin Nikolov --- .../neo4j/gds/paths/dijkstra/Dijkstra.java | 1 + .../gds/paths/yens/MutablePathResult.java | 8 +- .../java/org/neo4j/gds/paths/yens/Yens.java | 25 ++++- .../gds/paths/yens/MutablePathResultTest.java | 6 +- .../ShortestPathYensProc2Test.java | 103 ++++++++++++++++++ 5 files changed, 131 insertions(+), 12 deletions(-) create mode 100644 proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensProc2Test.java diff --git a/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java b/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java index 203721de43d..223d9c166e8 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java +++ b/algo/src/main/java/org/neo4j/gds/paths/dijkstra/Dijkstra.java @@ -217,6 +217,7 @@ private PathResult next(TraversalPredicate traversalPredicate, ImmutablePathResu return pathResult(node, pathResultBuilder); } } + return PathResult.EMPTY; } diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java b/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java index 27efd1e401c..7e4f244681b 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java @@ -131,10 +131,10 @@ boolean matches(MutablePathResult path, int index) { */ boolean matchesExactly(MutablePathResult path, int index) { - if (relationshipIds == null || path.relationshipIds == null) { + if (relationshipIds.length == 0 || path.relationshipIds.length == 0) { return matches(path, index); } - + //System.out.println(Arrays.toString(Arrays.stream(relationshipIds).toArray())); for (int i = 0; i < index; i++) { if (nodeIds[i] != path.nodeIds[i]) { return false; @@ -157,7 +157,7 @@ boolean matchesExactly(MutablePathResult path, int index) { * The cost value associated with the last value in this path, is added to * the costs for each node in the second path. */ - void append(MutablePathResult path) { + void append(MutablePathResult path, boolean shouldstore) { // spur node is end of first and beginning of second path assert nodeIds[nodeIds.length - 1] == path.nodeIds[0]; @@ -186,7 +186,7 @@ void append(MutablePathResult path) { } this.nodeIds = newNodeIds; - this.relationshipIds = newRelationshipIds; + this.relationshipIds = shouldstore ? newRelationshipIds : new long[0]; this.costs = newCosts; } diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index 2bd70bbb9a6..444603aced7 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -65,6 +65,16 @@ public static Yens sourceTarget( // If the input graph is a multi-graph, we need to track // parallel relationships. This is necessary since shortest // paths can visit the same nodes via different relationships. + + System.out.println(graph.schema().relationshipSchema().toMap()); + graph.forEachNode(nodeId -> { + graph.forEachRelationship(nodeId, 1.0, (s, t, w) -> { + System.out.println(s + "-[" + w + "]->" + t); + return true; + }); + return true; + }); + var newConfig = ImmutableShortestPathYensBaseConfig .builder() .from(config) @@ -101,10 +111,11 @@ private Yens(Graph graph, Dijkstra dijkstra, ShortestPathYensBaseConfig config, this.dijkstra = dijkstra; dijkstra.withRelationshipFilter((source, target, relationshipId) -> !nodeBlackList.contains(target) && - !(relationshipBlackList.getOrDefault(source, EMPTY_SET).contains(relationshipId)) - ); + !(relationshipBlackList.getOrDefault(source, EMPTY_SET).contains(relationshipId)) && + !(relationshipBlackList.getOrDefault(source, EMPTY_SET).contains(-target - 1))); } + @Override public DijkstraResult compute() { progressTracker.beginSubTask(); @@ -139,7 +150,8 @@ public DijkstraResult compute() { // Filter relationships that are part of the previous // shortest paths which share the same root path. if (rootPath.matchesExactly(path, n + 1)) { - var relationshipId = path.relationship(n); + System.out.println(i + ": " + rootPath + " |" + prevPath); + var relationshipId = graph.isMultiGraph() ? path.relationship(n) : -(1 + path.node(n + 1)); var neighbors = relationshipBlackList.get(spurNode); @@ -171,7 +183,7 @@ public DijkstraResult compute() { } // Entire path is made up of the root path and spur path. - rootPath.append(MutablePathResult.of(spurPath.get())); + rootPath.append(MutablePathResult.of(spurPath.get()), graph.isMultiGraph()); // Add the potential k-shortest path to the heap. if (!candidates.contains(rootPath)) { candidates.add(rootPath); @@ -189,7 +201,10 @@ public DijkstraResult compute() { progressTracker.endSubTask(); progressTracker.endSubTask(); - + System.out.println("----"); + for (var path : kShortestPaths) { + System.out.println(path); + } return new DijkstraResult(kShortestPaths.stream().map(MutablePathResult::toPathResult)); } diff --git a/algo/src/test/java/org/neo4j/gds/paths/yens/MutablePathResultTest.java b/algo/src/test/java/org/neo4j/gds/paths/yens/MutablePathResultTest.java index bce9b465bfb..7fa938e56a8 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/yens/MutablePathResultTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/yens/MutablePathResultTest.java @@ -170,7 +170,7 @@ void append() { var path1 = testPath(0, 1, 2); var path2 = testPath(2, 3, 4); - path1.append(path2); + path1.append(path2, true); var expected = testPath(0, 1, 2, 3, 4); assertEquals(expected, path1); @@ -184,7 +184,7 @@ void appendWithOffset() { var expected = MutablePathResult.of(ImmutablePathResult.builder().nodeIds(0, 1, 2, 3, 4).relationshipIds(0, 1, 2, 3).costs(0, 1, 42, 55, 79).sourceNode(0).targetNode(2).index(2).build()); //@formatter:on - p1.append(p2); + p1.append(p2, true); assertEquals(expected, p1); assertEquals(expected.totalCost(), p1.totalCost()); @@ -195,6 +195,6 @@ void appendFailedAssertion() { var path1 = testPath(0, 1, 2); var path2 = testPath(4, 3, 5); - assertThatThrownBy(() -> path1.append(path2)).isInstanceOf(AssertionError.class); + assertThatThrownBy(() -> path1.append(path2, true)).isInstanceOf(AssertionError.class); } } diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensProc2Test.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensProc2Test.java new file mode 100644 index 00000000000..b88dba61fba --- /dev/null +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensProc2Test.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.paths.sourcetarget; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.GdsCypher; +import org.neo4j.gds.RelationshipProjection; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.core.Aggregation; +import org.neo4j.gds.extension.Neo4jGraph; + +class ShortestPathYensProc2Tes extends BaseProcTest{ + + @Neo4jGraph + private static final String DB_CYPHER = + "CREATE (a:CITY), " + + "(b:CITY), " + + "(c:CITY), " + + "(d:CITY), " + + "(e:CITY), " + + "(f:CITY), " + + "(a)-[:ROAD]->(b), " + + "(a)-[:ROAD]->(b), " + + "(b)-[:ROAD]->(c), " + + "(b)-[:ROAD]->(d), " + + "(c)-[:ROAD]->(f), " + + "(d)-[:ROAD]->(e), " + + "(e)-[:ROAD]->(c), " + + "(e)-[:ROAD]->(f), " + + "(a)-[:PATH]->(b), " + + "(d)-[:PATH]->(e), " + + "(d)-[:PATH]->(e)"; + + @BeforeEach + void setup() throws Exception { + registerProcedures( + ShortestPathYensStreamProc.class, + GraphProjectProc.class + ); + + runQuery(GdsCypher.call("graph") + .graphProject() + .withAnyLabel() + .withRelationshipType("TYPE", RelationshipProjection.builder().type("*").aggregation(Aggregation.SINGLE).build()) + .yields()); + } + + @Test + void foo() { + runQuery("" + + "MATCH (source), (target) " + + "WHERE id(source)=0 AND id(target)=5 " + + "CALL gds.shortestPath.yens.stream(" + + " 'graph', " + + " {sourceNode:source, targetNode:target, k:3} " + + ") " + + "YIELD sourceNode " + + "RETURN *" + ); + + + + } + @Test + void bar() { + runQuery("CALL gds.graph.project.cypher(\n" + + " 'graphSingleType_NP',\n" + + " 'MATCH (n) RETURN id(n) AS id',\n" + + " 'MATCH (n)-[r]->(m) RETURN DISTINCT id(n) AS source, id(m) AS target'\n" + + ")"); + + runQuery("" + + "MATCH (source), (target) " + + "WHERE id(source)=0 AND id(target)=5 " + + "CALL gds.shortestPath.yens.stream(" + + " 'graphSingleType_NP', " + + " {sourceNode:source, targetNode:target, k:3} " + + ") " + + "YIELD sourceNode " + + "RETURN *" + ); + } +} + From ad92943cff71c8fa43af99dd1055efd9cd74ca12 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Feb 2023 14:52:07 +0100 Subject: [PATCH 190/400] Rewrite test Co-authored-by: Veselin Nikolov --- .../ShortestPathYensProc2Test.java | 103 ----------------- .../YensTestWithDifferentProjections.java | 104 ++++++++++++++++++ 2 files changed, 104 insertions(+), 103 deletions(-) delete mode 100644 proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensProc2Test.java create mode 100644 proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensProc2Test.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensProc2Test.java deleted file mode 100644 index b88dba61fba..00000000000 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathYensProc2Test.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.paths.sourcetarget; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.neo4j.gds.BaseProcTest; -import org.neo4j.gds.GdsCypher; -import org.neo4j.gds.RelationshipProjection; -import org.neo4j.gds.catalog.GraphProjectProc; -import org.neo4j.gds.core.Aggregation; -import org.neo4j.gds.extension.Neo4jGraph; - -class ShortestPathYensProc2Tes extends BaseProcTest{ - - @Neo4jGraph - private static final String DB_CYPHER = - "CREATE (a:CITY), " + - "(b:CITY), " + - "(c:CITY), " + - "(d:CITY), " + - "(e:CITY), " + - "(f:CITY), " + - "(a)-[:ROAD]->(b), " + - "(a)-[:ROAD]->(b), " + - "(b)-[:ROAD]->(c), " + - "(b)-[:ROAD]->(d), " + - "(c)-[:ROAD]->(f), " + - "(d)-[:ROAD]->(e), " + - "(e)-[:ROAD]->(c), " + - "(e)-[:ROAD]->(f), " + - "(a)-[:PATH]->(b), " + - "(d)-[:PATH]->(e), " + - "(d)-[:PATH]->(e)"; - - @BeforeEach - void setup() throws Exception { - registerProcedures( - ShortestPathYensStreamProc.class, - GraphProjectProc.class - ); - - runQuery(GdsCypher.call("graph") - .graphProject() - .withAnyLabel() - .withRelationshipType("TYPE", RelationshipProjection.builder().type("*").aggregation(Aggregation.SINGLE).build()) - .yields()); - } - - @Test - void foo() { - runQuery("" + - "MATCH (source), (target) " + - "WHERE id(source)=0 AND id(target)=5 " + - "CALL gds.shortestPath.yens.stream(" + - " 'graph', " + - " {sourceNode:source, targetNode:target, k:3} " + - ") " + - "YIELD sourceNode " + - "RETURN *" - ); - - - - } - @Test - void bar() { - runQuery("CALL gds.graph.project.cypher(\n" + - " 'graphSingleType_NP',\n" + - " 'MATCH (n) RETURN id(n) AS id',\n" + - " 'MATCH (n)-[r]->(m) RETURN DISTINCT id(n) AS source, id(m) AS target'\n" + - ")"); - - runQuery("" + - "MATCH (source), (target) " + - "WHERE id(source)=0 AND id(target)=5 " + - "CALL gds.shortestPath.yens.stream(" + - " 'graphSingleType_NP', " + - " {sourceNode:source, targetNode:target, k:3} " + - ") " + - "YIELD sourceNode " + - "RETURN *" - ); - } -} - diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java new file mode 100644 index 00000000000..51f2e2b6045 --- /dev/null +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.paths.sourcetarget; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.Neo4jGraph; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class YensTestWithDifferentProjections extends BaseProcTest { + + @Neo4jGraph + private static final String DB_CYPHER = + "CREATE (a:CITY), " + + "(b:CITY), " + + "(c:CITY), " + + "(d:CITY), " + + "(e:CITY), " + + "(f:CITY), " + + "(a)-[:ROAD]->(b), " + + "(a)-[:ROAD]->(b), " + + "(b)-[:ROAD]->(c), " + + "(b)-[:ROAD]->(d), " + + "(c)-[:ROAD]->(f), " + + "(d)-[:ROAD]->(e), " + + "(e)-[:ROAD]->(c), " + + "(e)-[:ROAD]->(f), " + + "(a)-[:PATH]->(b), " + + "(d)-[:PATH]->(e), " + + "(d)-[:PATH]->(e)"; + + @BeforeEach + void setup() throws Exception { + registerProcedures( + ShortestPathYensStreamProc.class, + GraphProjectProc.class + ); + } + + + @ParameterizedTest + @ValueSource(strings = { + "CALL gds.graph.project('g', '*', {TYPE: {type: '*', aggregation: 'SINGLE'}})", + "CALL gds.graph.project.cypher('g', 'MATCH (n) RETURN id(n) AS id', 'MATCH (n)-[r]->(m) RETURN DISTINCT id(n) AS source, id(m) AS target')" + }) + void shouldWorkWithDifferentProjections(String projectionQuery) { + + runQuery(projectionQuery); + String yensQuery = "MATCH (source), (target) " + + "WHERE id(source)=0 AND id(target)=5 " + + "CALL gds.shortestPath.yens.stream(" + + " 'g', " + + " {sourceNode:source, targetNode:target, k:3} " + + ") " + + "YIELD nodeIds RETURN nodeIds "; + + Collection encounteredPaths = new HashSet<>(); + runQuery(yensQuery, result -> { + assertThat(result.columns()).containsExactlyInAnyOrder("nodeIds"); + + while (result.hasNext()) { + var next = result.next(); + var currentPath = (List) next.get("nodeIds"); + long[] pathToArray = currentPath.stream().mapToLong(l -> l).toArray(); + encounteredPaths.add(pathToArray); + } + + return true; + }); + + assertThat(encounteredPaths).containsExactlyInAnyOrder( + new long[]{0l, 1l, 3l, 4l, 2l, 5l}, + new long[]{0l, 1l, 3l, 4l, 5l}, + new long[]{0l, 1l, 2l, 5l} + ); + } + +} + From c90296f717431cd1499d35c76161d28eed88363a Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Feb 2023 15:32:05 +0100 Subject: [PATCH 191/400] Tests for not-multigraphs do not track relationships Co-authored-by: Veselin Nikolov --- .../org/neo4j/gds/paths/yens/YensTest.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java b/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java index c3af4a3b569..0b81c5bea25 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java @@ -32,6 +32,7 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.Aggregation; import org.neo4j.gds.core.utils.mem.MemoryRange; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -85,7 +86,7 @@ void shouldComputeMemoryEstimation(int nodeCount, long expectedBytes) { } // https://en.wikipedia.org/wiki/Yen%27s_algorithm#/media/File:Yen's_K-Shortest_Path_Algorithm,_K=3,_A_to_F.gif - @GdlGraph + @GdlGraph(aggregation = Aggregation.SINGLE) private static final String DB_CYPHER = "CREATE" + " (c:C {id: 0})" + @@ -165,7 +166,8 @@ static Stream> pathInput() { @ParameterizedTest @MethodSource("pathInput") void compute(Collection expectedPaths) { - assertResult(graph, idFunction, expectedPaths); + + assertResult(graph, idFunction, expectedPaths, false); } @Test @@ -238,8 +240,13 @@ void shouldCloseProgressTasksOnEmptyResult() { ); } - private static void assertResult(Graph graph, IdFunction idFunction, Collection expectedPaths) { - var expectedPathResults = expectedPathResults(idFunction, expectedPaths); + private static void assertResult( + Graph graph, + IdFunction idFunction, + Collection expectedPaths, + boolean trackRelationships + ) { + var expectedPathResults = expectedPathResults(idFunction, expectedPaths, trackRelationships); var firstResult = expectedPathResults .stream() @@ -267,7 +274,11 @@ private static void assertResult(Graph graph, IdFunction idFunction, Collection< } @NotNull - private static Set expectedPathResults(IdFunction idFunction, Collection expectedPaths) { + private static Set expectedPathResults( + IdFunction idFunction, + Collection expectedPaths, + boolean trackRelationships + ) { var index = new MutableInt(0); return expectedPaths.stream() .map(expectedPath -> new GDLHandler.Builder() @@ -312,7 +323,7 @@ private static Set expectedPathResults(IdFunction idFunction, Collec .sourceNode(sourceNode.getId()) .targetNode(targetNode.getId()) .nodeIds(nodeIds) - .relationshipIds(relationshipIds) + .relationshipIds(trackRelationships ? relationshipIds : new long[0]) .costs(costs) .build(); }) @@ -375,7 +386,7 @@ Stream> pathInput() { @ParameterizedTest @MethodSource("pathInput") void compute(Collection expectedPaths) { - assertResult(graph, idFunction, expectedPaths); + assertResult(graph, idFunction, expectedPaths, true); } } } From 7f578cebdcb0f58f1c5ba28c5338c7db300771ce Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Feb 2023 16:06:15 +0100 Subject: [PATCH 192/400] Refactor Yens Co-authored-by: Veselin Nikolov --- .../java/org/neo4j/gds/paths/yens/Yens.java | 72 +++++++++++-------- .../org/neo4j/gds/paths/yens/YensTest.java | 1 + 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index 444603aced7..50de0423af1 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -39,6 +39,7 @@ import java.util.Comparator; import java.util.Optional; import java.util.PriorityQueue; +import java.util.function.ToLongBiFunction; import java.util.stream.Stream; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -50,9 +51,10 @@ public final class Yens extends Algorithm { private final Graph graph; private final ShortestPathYensBaseConfig config; private final Dijkstra dijkstra; - - private final LongScatterSet nodeBlackList; - private final LongObjectScatterMap relationshipBlackList; + private final LongScatterSet nodeAvoidList; + private final LongObjectScatterMap relationshipAvoidList; + private final ToLongBiFunction + relationshipAvoidMapper; /** * Configure Yens to compute at most one source-target shortest path. @@ -63,22 +65,15 @@ public static Yens sourceTarget( ProgressTracker progressTracker ) { // If the input graph is a multi-graph, we need to track - // parallel relationships. This is necessary since shortest + // parallel relationships ids. This is necessary since shortest // paths can visit the same nodes via different relationships. + //If not, we need to track which is the next neighbor. - System.out.println(graph.schema().relationshipSchema().toMap()); - graph.forEachNode(nodeId -> { - graph.forEachRelationship(nodeId, 1.0, (s, t, w) -> { - System.out.println(s + "-[" + w + "]->" + t); - return true; - }); - return true; - }); - + boolean shouldTrackRelationships = graph.isMultiGraph(); var newConfig = ImmutableShortestPathYensBaseConfig .builder() .from(config) - .trackRelationships(graph.isMultiGraph()) + .trackRelationships(shouldTrackRelationships) .build(); // Init dijkstra algorithm for computing shortest paths var dijkstra = Dijkstra.sourceTarget(graph, newConfig, Optional.empty(), progressTracker); @@ -105,16 +100,35 @@ private Yens(Graph graph, Dijkstra dijkstra, ShortestPathYensBaseConfig config, this.config = config; // Track nodes and relationships that are skipped in a single iteration. // The content of these data structures is reset after each of k iterations. - this.nodeBlackList = new LongScatterSet(); - this.relationshipBlackList = new LongObjectScatterMap<>(); - // set filter in Dijkstra to respect our blacklists + this.nodeAvoidList = new LongScatterSet(); + this.relationshipAvoidList = new LongObjectScatterMap<>(); + // set filter in Dijkstra to respect our list of relationships to avoid this.dijkstra = dijkstra; + + if (config.trackRelationships()) { + // if we are in a multi-graph, we must store the relationships ids as they are + //since two nodes may be connected by multiple relationships and we must know which to avoid + relationshipAvoidMapper = (path, position) -> path.relationship(position); + } else { + //otherwise the graph has surely no parallel edges, we do not need to explicitly store relationship ids + //we can just store endpoints, so that we know which nodes a node should avoid + relationshipAvoidMapper = (path, position) -> path.node(position + 1); + } dijkstra.withRelationshipFilter((source, target, relationshipId) -> - !nodeBlackList.contains(target) && - !(relationshipBlackList.getOrDefault(source, EMPTY_SET).contains(relationshipId)) && - !(relationshipBlackList.getOrDefault(source, EMPTY_SET).contains(-target - 1))); + !nodeAvoidList.contains(target) + && !shouldAvoidRelationship(source, target, relationshipId) + + ); } + private boolean shouldAvoidRelationship(long source, long target, long relationshipId) { + long forbidden = target; + if (config.trackRelationships()) { + forbidden = relationshipId; + } + return relationshipAvoidList.getOrDefault(source, EMPTY_SET).contains(forbidden); + + } @Override public DijkstraResult compute() { @@ -150,14 +164,13 @@ public DijkstraResult compute() { // Filter relationships that are part of the previous // shortest paths which share the same root path. if (rootPath.matchesExactly(path, n + 1)) { - System.out.println(i + ": " + rootPath + " |" + prevPath); - var relationshipId = graph.isMultiGraph() ? path.relationship(n) : -(1 + path.node(n + 1)); + var relationshipId = relationshipAvoidMapper.applyAsLong(path, n); - var neighbors = relationshipBlackList.get(spurNode); + var neighbors = relationshipAvoidList.get(spurNode); if (neighbors == null) { neighbors = new LongHashSet(); - relationshipBlackList.put(spurNode, neighbors); + relationshipAvoidList.put(spurNode, neighbors); } neighbors.add(relationshipId); } @@ -165,7 +178,7 @@ public DijkstraResult compute() { // Filter nodes from root path to avoid cyclic path searches. for (int j = 0; j < n; j++) { - nodeBlackList.add(rootPath.node(j)); + nodeAvoidList.add(rootPath.node(j)); } // Calculate the spur path from the spur node to the sink. @@ -174,8 +187,8 @@ public DijkstraResult compute() { var spurPath = computeDijkstra(graph.toOriginalNodeId(spurNode)); // Clear filters for next spur node - nodeBlackList.clear(); - relationshipBlackList.clear(); + nodeAvoidList.clear(); + relationshipAvoidList.clear(); // No new candidate from this spur node, continue with next node. if (spurPath.isEmpty()) { @@ -201,10 +214,7 @@ public DijkstraResult compute() { progressTracker.endSubTask(); progressTracker.endSubTask(); - System.out.println("----"); - for (var path : kShortestPaths) { - System.out.println(path); - } + return new DijkstraResult(kShortestPaths.stream().map(MutablePathResult::toPathResult)); } diff --git a/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java b/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java index 0b81c5bea25..4dc677e6e4f 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java @@ -168,6 +168,7 @@ static Stream> pathInput() { void compute(Collection expectedPaths) { assertResult(graph, idFunction, expectedPaths, false); + } @Test From 3091f915effcb769085dd6ec0827bb6183a7a262 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Feb 2023 16:29:13 +0100 Subject: [PATCH 193/400] Refactor MutablePathResult Co-authored-by: Veselin Nikolov --- .../gds/paths/yens/MutablePathResult.java | 45 +++++++++++++++---- .../java/org/neo4j/gds/paths/yens/Yens.java | 10 ++++- .../gds/paths/yens/MutablePathResultTest.java | 6 +-- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java b/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java index 7e4f244681b..f973d792308 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java @@ -134,7 +134,6 @@ boolean matchesExactly(MutablePathResult path, int index) { if (relationshipIds.length == 0 || path.relationshipIds.length == 0) { return matches(path, index); } - //System.out.println(Arrays.toString(Arrays.stream(relationshipIds).toArray())); for (int i = 0; i < index; i++) { if (nodeIds[i] != path.nodeIds[i]) { return false; @@ -157,7 +156,9 @@ boolean matchesExactly(MutablePathResult path, int index) { * The cost value associated with the last value in this path, is added to * the costs for each node in the second path. */ - void append(MutablePathResult path, boolean shouldstore) { + + + private void append(MutablePathResult path, long[] relationships) { // spur node is end of first and beginning of second path assert nodeIds[nodeIds.length - 1] == path.nodeIds[0]; @@ -166,15 +167,10 @@ void append(MutablePathResult path, boolean shouldstore) { var newNodeIds = new long[oldLength + path.nodeIds.length - 1]; var newCosts = new double[oldLength + path.nodeIds.length - 1]; - var oldRelationshipIdsLength = relationshipIds.length; - var newRelationshipIds = new long[oldRelationshipIdsLength + path.relationshipIds.length]; - // copy node ids System.arraycopy(this.nodeIds, 0, newNodeIds, 0, oldLength); System.arraycopy(path.nodeIds, 1, newNodeIds, oldLength, path.nodeIds.length - 1); - // copy relationship ids - System.arraycopy(this.relationshipIds, 0, newRelationshipIds, 0, oldRelationshipIdsLength); - System.arraycopy(path.relationshipIds, 0, newRelationshipIds, oldRelationshipIdsLength, path.relationshipIds.length); + // copy costs System.arraycopy(this.costs, 0, newCosts, 0, oldLength); System.arraycopy(path.costs, 1, newCosts, oldLength, path.costs.length - 1); @@ -186,10 +182,41 @@ void append(MutablePathResult path, boolean shouldstore) { } this.nodeIds = newNodeIds; - this.relationshipIds = shouldstore ? newRelationshipIds : new long[0]; + this.relationshipIds = relationships; this.costs = newCosts; } + void append(MutablePathResult path) { + + var oldRelationshipIdsLength = relationshipIds.length; + var newRelationshipIds = new long[oldRelationshipIdsLength + path.relationshipIds.length]; + // copy relationship ids + System.arraycopy(this.relationshipIds, 0, newRelationshipIds, 0, oldRelationshipIdsLength); + System.arraycopy( + path.relationshipIds, + 0, + newRelationshipIds, + oldRelationshipIdsLength, + path.relationshipIds.length + ); + + append(path, newRelationshipIds); + } + + /** + * Appends the given path to this path without creating an explicit relationship array. + * + * The last node in this path, must match the first node in the given path. + * This node will only appear once in the resulting path. + * The cost value associated with the last value in this path, is added to + * the costs for each node in the second path. + */ + void appendWithoutRelationshipIds(MutablePathResult path) { + // spur node is end of first and beginning of second path + append(path, new long[0]); + } + + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index 50de0423af1..a2052a57f55 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -39,6 +39,7 @@ import java.util.Comparator; import java.util.Optional; import java.util.PriorityQueue; +import java.util.function.BiConsumer; import java.util.function.ToLongBiFunction; import java.util.stream.Stream; @@ -55,6 +56,8 @@ public final class Yens extends Algorithm { private final LongObjectScatterMap relationshipAvoidList; private final ToLongBiFunction relationshipAvoidMapper; + private final BiConsumer pathAppender; + /** * Configure Yens to compute at most one source-target shortest path. @@ -109,10 +112,12 @@ private Yens(Graph graph, Dijkstra dijkstra, ShortestPathYensBaseConfig config, // if we are in a multi-graph, we must store the relationships ids as they are //since two nodes may be connected by multiple relationships and we must know which to avoid relationshipAvoidMapper = (path, position) -> path.relationship(position); + pathAppender = (rootPath, spurPath) -> rootPath.append(MutablePathResult.of(spurPath)); } else { //otherwise the graph has surely no parallel edges, we do not need to explicitly store relationship ids //we can just store endpoints, so that we know which nodes a node should avoid relationshipAvoidMapper = (path, position) -> path.node(position + 1); + pathAppender = (rootPath, spurPath) -> rootPath.appendWithoutRelationshipIds(MutablePathResult.of(spurPath)); } dijkstra.withRelationshipFilter((source, target, relationshipId) -> !nodeAvoidList.contains(target) @@ -196,7 +201,8 @@ public DijkstraResult compute() { } // Entire path is made up of the root path and spur path. - rootPath.append(MutablePathResult.of(spurPath.get()), graph.isMultiGraph()); + pathAppender.accept(rootPath, spurPath.get()); + // Add the potential k-shortest path to the heap. if (!candidates.contains(rootPath)) { candidates.add(rootPath); @@ -214,7 +220,7 @@ public DijkstraResult compute() { progressTracker.endSubTask(); progressTracker.endSubTask(); - + return new DijkstraResult(kShortestPaths.stream().map(MutablePathResult::toPathResult)); } diff --git a/algo/src/test/java/org/neo4j/gds/paths/yens/MutablePathResultTest.java b/algo/src/test/java/org/neo4j/gds/paths/yens/MutablePathResultTest.java index 7fa938e56a8..bce9b465bfb 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/yens/MutablePathResultTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/yens/MutablePathResultTest.java @@ -170,7 +170,7 @@ void append() { var path1 = testPath(0, 1, 2); var path2 = testPath(2, 3, 4); - path1.append(path2, true); + path1.append(path2); var expected = testPath(0, 1, 2, 3, 4); assertEquals(expected, path1); @@ -184,7 +184,7 @@ void appendWithOffset() { var expected = MutablePathResult.of(ImmutablePathResult.builder().nodeIds(0, 1, 2, 3, 4).relationshipIds(0, 1, 2, 3).costs(0, 1, 42, 55, 79).sourceNode(0).targetNode(2).index(2).build()); //@formatter:on - p1.append(p2, true); + p1.append(p2); assertEquals(expected, p1); assertEquals(expected.totalCost(), p1.totalCost()); @@ -195,6 +195,6 @@ void appendFailedAssertion() { var path1 = testPath(0, 1, 2); var path2 = testPath(4, 3, 5); - assertThatThrownBy(() -> path1.append(path2, true)).isInstanceOf(AssertionError.class); + assertThatThrownBy(() -> path1.append(path2)).isInstanceOf(AssertionError.class); } } From efbc06e6503485411d13234a220ebb0273894701 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 13 Feb 2023 15:36:16 +0100 Subject: [PATCH 194/400] Addressing review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Florentin Dörre --- .../gds/paths/yens/MutablePathResult.java | 6 +- .../java/org/neo4j/gds/paths/yens/Yens.java | 7 +- .../org/neo4j/gds/paths/yens/YensTest.java | 2 - .../YensTestWithDifferentProjections.java | 137 +++++++++--------- 4 files changed, 78 insertions(+), 74 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java b/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java index f973d792308..8d5e43eb081 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/MutablePathResult.java @@ -31,6 +31,7 @@ */ final class MutablePathResult { + private final long[] EMPTY_ARRAY = new long[0]; private long index; private final long sourceNode; @@ -156,8 +157,7 @@ boolean matchesExactly(MutablePathResult path, int index) { * The cost value associated with the last value in this path, is added to * the costs for each node in the second path. */ - - + private void append(MutablePathResult path, long[] relationships) { // spur node is end of first and beginning of second path assert nodeIds[nodeIds.length - 1] == path.nodeIds[0]; @@ -213,7 +213,7 @@ void append(MutablePathResult path) { */ void appendWithoutRelationshipIds(MutablePathResult path) { // spur node is end of first and beginning of second path - append(path, new long[0]); + append(path, EMPTY_ARRAY); } diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index a2052a57f55..048878666fa 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -127,10 +127,9 @@ private Yens(Graph graph, Dijkstra dijkstra, ShortestPathYensBaseConfig config, } private boolean shouldAvoidRelationship(long source, long target, long relationshipId) { - long forbidden = target; - if (config.trackRelationships()) { - forbidden = relationshipId; - } + long forbidden = config.trackRelationships() + ? relationshipId + : target; return relationshipAvoidList.getOrDefault(source, EMPTY_SET).contains(forbidden); } diff --git a/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java b/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java index 4dc677e6e4f..6275662143e 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/yens/YensTest.java @@ -166,9 +166,7 @@ static Stream> pathInput() { @ParameterizedTest @MethodSource("pathInput") void compute(Collection expectedPaths) { - assertResult(graph, idFunction, expectedPaths, false); - } @Test diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java index 51f2e2b6045..a176c6aa5de 100644 --- a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/YensTestWithDifferentProjections.java @@ -24,6 +24,8 @@ import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.BaseProcTest; import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.extension.IdFunction; +import org.neo4j.gds.extension.Inject; import org.neo4j.gds.extension.Neo4jGraph; import java.util.Collection; @@ -34,71 +36,76 @@ class YensTestWithDifferentProjections extends BaseProcTest { - @Neo4jGraph - private static final String DB_CYPHER = - "CREATE (a:CITY), " + - "(b:CITY), " + - "(c:CITY), " + - "(d:CITY), " + - "(e:CITY), " + - "(f:CITY), " + - "(a)-[:ROAD]->(b), " + - "(a)-[:ROAD]->(b), " + - "(b)-[:ROAD]->(c), " + - "(b)-[:ROAD]->(d), " + - "(c)-[:ROAD]->(f), " + - "(d)-[:ROAD]->(e), " + - "(e)-[:ROAD]->(c), " + - "(e)-[:ROAD]->(f), " + - "(a)-[:PATH]->(b), " + - "(d)-[:PATH]->(e), " + - "(d)-[:PATH]->(e)"; - - @BeforeEach - void setup() throws Exception { - registerProcedures( - ShortestPathYensStreamProc.class, - GraphProjectProc.class - ); - } - - - @ParameterizedTest - @ValueSource(strings = { - "CALL gds.graph.project('g', '*', {TYPE: {type: '*', aggregation: 'SINGLE'}})", - "CALL gds.graph.project.cypher('g', 'MATCH (n) RETURN id(n) AS id', 'MATCH (n)-[r]->(m) RETURN DISTINCT id(n) AS source, id(m) AS target')" - }) - void shouldWorkWithDifferentProjections(String projectionQuery) { - - runQuery(projectionQuery); - String yensQuery = "MATCH (source), (target) " + - "WHERE id(source)=0 AND id(target)=5 " + - "CALL gds.shortestPath.yens.stream(" + - " 'g', " + - " {sourceNode:source, targetNode:target, k:3} " + - ") " + - "YIELD nodeIds RETURN nodeIds "; - - Collection encounteredPaths = new HashSet<>(); - runQuery(yensQuery, result -> { - assertThat(result.columns()).containsExactlyInAnyOrder("nodeIds"); - - while (result.hasNext()) { - var next = result.next(); - var currentPath = (List) next.get("nodeIds"); - long[] pathToArray = currentPath.stream().mapToLong(l -> l).toArray(); - encounteredPaths.add(pathToArray); - } - - return true; - }); - - assertThat(encounteredPaths).containsExactlyInAnyOrder( - new long[]{0l, 1l, 3l, 4l, 2l, 5l}, - new long[]{0l, 1l, 3l, 4l, 5l}, - new long[]{0l, 1l, 2l, 5l} - ); - } + @Neo4jGraph + private static final String DB_CYPHER = + "CREATE (a:CITY {cityid:0}), " + + "(b:CITY {cityid:1}), " + + "(c:CITY {cityid:2}), " + + "(d:CITY {cityid:3}), " + + "(e:CITY {cityid:4}), " + + "(f:CITY {cityid:5}), " + + "(a)-[:ROAD]->(b), " + + "(a)-[:ROAD]->(b), " + + "(b)-[:ROAD]->(c), " + + "(b)-[:ROAD]->(d), " + + "(c)-[:ROAD]->(f), " + + "(d)-[:ROAD]->(e), " + + "(e)-[:ROAD]->(c), " + + "(e)-[:ROAD]->(f), " + + "(a)-[:PATH]->(b), " + + "(d)-[:PATH]->(e), " + + "(d)-[:PATH]->(e)"; + + @Inject + IdFunction idFunction; + + @BeforeEach + void setup() throws Exception { + registerProcedures( + ShortestPathYensStreamProc.class, + GraphProjectProc.class + ); + } + + + @ParameterizedTest + @ValueSource(strings = { + "CALL gds.graph.project('g', '*', {TYPE: {type: '*', aggregation: 'SINGLE'}})", + "CALL gds.graph.project.cypher('g', 'MATCH (n) RETURN id(n) AS id', 'MATCH (n)-[r]->(m) RETURN DISTINCT id(n) AS source, id(m) AS target')" + }) + void shouldWorkWithDifferentProjections(String projectionQuery) { + + runQuery(projectionQuery); + String yensQuery = "MATCH (source), (target) " + + "WHERE source.cityid=0 AND target.cityid=5 " + + "CALL gds.shortestPath.yens.stream(" + + " 'g', " + + " {sourceNode:source, targetNode:target, k:3} " + + ") " + + "YIELD nodeIds RETURN nodeIds "; + + Collection encounteredPaths = new HashSet<>(); + runQuery(yensQuery, result -> { + assertThat(result.columns()).containsExactlyInAnyOrder("nodeIds"); + + while (result.hasNext()) { + var next = result.next(); + var currentPath = (List) next.get("nodeIds"); + long[] pathToArray = currentPath.stream().mapToLong(l -> l).toArray(); + encounteredPaths.add(pathToArray); + } + + return true; + }); + + long[] nodes = new long[]{idFunction.of("a"), idFunction.of("b"), idFunction.of("c"), idFunction.of("d"), idFunction.of( + "e"), idFunction.of("f")}; + assertThat(encounteredPaths).containsExactlyInAnyOrder( + new long[]{nodes[0], nodes[1], nodes[3], nodes[4], nodes[2], nodes[5]}, + new long[]{nodes[0], nodes[1], nodes[3], nodes[4], nodes[5]}, + new long[]{nodes[0], nodes[1], nodes[2], nodes[5]} + ); + } } From 412ed2a43ef67ace598a6956f7647fbe2de3534f Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 14 Feb 2023 07:49:40 +0100 Subject: [PATCH 195/400] Extra treatment for 2.3 branch --- algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java index 048878666fa..8ecc2700b17 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java +++ b/algo/src/main/java/org/neo4j/gds/paths/yens/Yens.java @@ -233,8 +233,8 @@ private PriorityQueue initCandidatesQueue() { @Override public void release() { dijkstra.release(); - nodeBlackList.release(); - relationshipBlackList.release(); + nodeAvoidList.release(); + relationshipAvoidList.release(); } private Optional computeDijkstra(long sourceNode) { From 271203b6ee98bc89a0c0b91048b7802e32850c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 16 Feb 2023 14:50:12 +0100 Subject: [PATCH 196/400] Update to 2.3.2[+13] Co-Authored-By: Paul Horn --- README.adoc | 19 ++++++++++--------- .../management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 4 ++-- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/README.adoc b/README.adoc index a3348eb72fa..2ac5017b210 100644 --- a/README.adoc +++ b/README.adoc @@ -83,12 +83,13 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.2.0 |Neo4j 5.3.0 |Neo4j 5.4.0 -.5+<.^|GDS 2.3.x +.6+<.^|GDS 2.3.x |Neo4j 4.4.9 - 4.4.17 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 |Neo4j 5.4.0 +|Neo4j 5.5.0 |=== NOTE: Preview releases are not automatically made available in Neo4j Desktop. They need to be installed manually. @@ -132,7 +133,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.3.0 + 2.3.2 ---- @@ -144,21 +145,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.0 + 2.3.2 org.neo4j.gds algo - 2.3.0 + 2.3.2 org.neo4j.gds alpha-algo - 2.3.0 + 2.3.2 ---- @@ -170,28 +171,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.0 + 2.3.2 org.neo4j.gds proc - 2.3.0 + 2.3.2 org.neo4j.gds alpha-proc - 2.3.0 + 2.3.2 org.neo4j.gds write-services - 2.3.0 + 2.3.2 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 4bd288497cc..5a7a4c839a7 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.1" +| "2.3.2" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index e5e2a3aa3f0..85af697057e 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.0' + gdsVersion = '2.3.2' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index 50b45e7cfd5..651ff77fa79 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { - gdsBaseVersion = '2.3.1' - gdsAuraVersion = '12' + gdsBaseVersion = '2.3.2' + gdsAuraVersion = '13' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 9febf19a10ecf8d8e464aa3d8b8398eb49b82e2c Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 20 Feb 2023 13:39:49 +0100 Subject: [PATCH 197/400] Fix output yields issue for kmeans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- .../main/java/org/neo4j/gds/executor/ExecutionContext.java | 4 +--- .../test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java b/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java index 6261eae4252..4da733a0961 100644 --- a/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java +++ b/executor/src/main/java/org/neo4j/gds/executor/ExecutionContext.java @@ -42,8 +42,6 @@ import org.neo4j.logging.Log; import org.neo4j.logging.NullLog; -import static org.neo4j.gds.utils.StringFormatting.toLowerCaseWithLocale; - // TODO Remove the @Nullable annotations once the EstimationCli uses ProcedureExecutors @ValueClass public interface ExecutionContext { @@ -96,7 +94,7 @@ default DatabaseId databaseId() { @Value.Lazy default boolean containsOutputField(String fieldName) { return callContext().outputFields() - .anyMatch(field -> toLowerCaseWithLocale(field).equals(fieldName)); + .anyMatch(field -> field.equals(fieldName)); } @Value.Lazy diff --git a/proc/community/src/test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java b/proc/community/src/test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java index 90bd4e0a1de..776bc64607c 100644 --- a/proc/community/src/test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/kmeans/KmeansStatsProcTest.java @@ -110,7 +110,7 @@ void yields() { assertThat(resultRow.get("averageDistanceToCentroid")) .isNotNull() - .asInstanceOf(DOUBLE); + .asInstanceOf(DOUBLE).isNotEqualTo(0.0d); var centroids = resultRow.get("centroids"); assertThat(centroids) @@ -126,7 +126,7 @@ void yields() { } assertThat(resultRow.get("averageSilhouette")) .isNotNull() - .asInstanceOf(DOUBLE).isGreaterThanOrEqualTo(-1d).isLessThanOrEqualTo(1d); + .asInstanceOf(DOUBLE).isGreaterThanOrEqualTo(-1d).isLessThanOrEqualTo(1d).isNotEqualTo(0.0d); } return true; From 18ad2ce00fb954d0b97c18881c495218e9861d05 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 21 Feb 2023 10:32:03 +0100 Subject: [PATCH 198/400] Configure Scala version for 5.6 --- gradle/dependencies.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index f76dfdc8378..fe93cdd994b 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -19,6 +19,7 @@ ext { '5.3': '2.13.8', '5.4': '2.13.8', '5.5': '2.13.8', + '5.6': '2.13.8', ] ver = [ From bbb2a07488c1635552cbfd00f05b3bbb0fca31c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 8 Feb 2023 12:28:46 +0100 Subject: [PATCH 199/400] Fix transaction dev compat issues --- .../_44/InMemoryTransactionIdStoreImpl.java | 81 ++++++++++++++++- .../_51/InMemoryTransactionIdStoreImpl.java | 23 ++++- .../_52/InMemoryTransactionIdStoreImpl.java | 24 ++++- .../_53/InMemoryTransactionIdStoreImpl.java | 24 ++++- .../_54/InMemoryTransactionIdStoreImpl.java | 24 ++++- .../_55/InMemoryMetaDataProviderImpl.java | 89 ++++++++++--------- .../_55/InMemoryTransactionIdStoreImpl.java | 72 ++++++++++----- .../AbstractTransactionIdStore.java | 86 +++++------------- 8 files changed, 284 insertions(+), 139 deletions(-) diff --git a/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java index 7f7818f3db6..98905edd331 100644 --- a/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryTransactionIdStoreImpl.java @@ -23,6 +23,7 @@ import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { @@ -32,7 +33,85 @@ public ClosedTransactionMetadata getLastClosedTransaction() { } @Override - protected CursorContext getEmptyCursorContext() { + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion, + getEmptyCursorContext() + ); + } + + @Override + public synchronized void transactionCommitted( + long transactionId, + int checksum, + long commitTimestamp, + CursorContext cursorContext + ) { + TransactionId current = this.committedTransactionId.get(); + if (current == null || transactionId > current.transactionId()) { + this.committedTransactionId.set(transactionId(transactionId, checksum, commitTimestamp)); + } + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long byteOffset, + long logVersion, + CursorContext cursorContext + ) { + this.committingTransactionId.set(transactionId); + this.committedTransactionId.set(transactionId(transactionId, checksum, commitTimestamp)); + this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long byteOffset, + long logVersion, + boolean missingLogs, + CursorContext cursorContext + ) { + this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset}); + } + + @Override + public TransactionId getUpgradeTransaction() { + return transactionId( + this.previouslyCommittedTxId, + this.initialTransactionChecksum, + this.previouslyCommittedTxCommitTimestamp + ); + } + + @Override + public void transactionClosed(long transactionId, long logVersion, long byteOffset, CursorContext cursorContext) { + this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset}); + } + + @Override + public void flush(CursorContext cursorContext) { + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); + } + + private CursorContext getEmptyCursorContext() { return CursorContext.NULL; } } diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java index 0ff4ec50d7c..f370df22453 100644 --- a/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryTransactionIdStoreImpl.java @@ -20,9 +20,9 @@ package org.neo4j.gds.compat._51; import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { @@ -36,6 +36,23 @@ public ClosedTransactionMetadata getLastClosedTransaction() { ); } + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + @Override public void transactionClosed( long transactionId, @@ -69,7 +86,7 @@ public void setLastCommittedAndClosedTransactionId( } @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL_CONTEXT; + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); } } diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java index 4932d715975..ea4880044c3 100644 --- a/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryTransactionIdStoreImpl.java @@ -20,12 +20,30 @@ package org.neo4j.gds.compat._52; import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override public ClosedTransactionMetadata getLastClosedTransaction() { long[] metaData = this.closedTransactionId.get(); return new ClosedTransactionMetadata( @@ -69,7 +87,7 @@ public void setLastCommittedAndClosedTransactionId( } @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL_CONTEXT; + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); } } diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java index 73749482bab..845f307c05c 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryTransactionIdStoreImpl.java @@ -20,12 +20,30 @@ package org.neo4j.gds.compat._53; import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override public ClosedTransactionMetadata getLastClosedTransaction() { long[] metaData = this.closedTransactionId.get(); return new ClosedTransactionMetadata( @@ -69,7 +87,7 @@ public void setLastCommittedAndClosedTransactionId( } @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL_CONTEXT; + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); } } diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java index 40f845337e4..3a0ee477755 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryTransactionIdStoreImpl.java @@ -20,12 +20,30 @@ package org.neo4j.gds.compat._54; import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override public ClosedTransactionMetadata getLastClosedTransaction() { long[] metaData = this.closedTransactionId.get(); return new ClosedTransactionMetadata( @@ -69,7 +87,7 @@ public void setLastCommittedAndClosedTransactionId( } @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL_CONTEXT; + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); } } diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java index 8762b580155..e551e1a75c1 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java @@ -26,7 +26,6 @@ import org.neo4j.storageengine.api.MetadataProvider; import org.neo4j.storageengine.api.StoreId; import org.neo4j.storageengine.api.TransactionId; -import org.neo4j.storageengine.api.TransactionIdStore; import java.io.IOException; import java.util.Optional; @@ -54,40 +53,6 @@ public ClosedTransactionMetadata getLastClosedTransaction() { return this.transactionIdStore.getLastClosedTransaction(); } - @Override - public void transactionClosed( - long transactionId, - long logVersion, - long byteOffset, - int checksum, - long commitTimestamp - ) { - this.transactionIdStore.transactionClosed( - transactionId, - logVersion, - byteOffset, - checksum, - commitTimestamp - ); - } - - @Override - public void resetLastClosedTransaction( - long transactionId, - long logVersion, - long byteOffset, - int checksum, - long commitTimestamp - ) { - this.transactionIdStore.resetLastClosedTransaction( - transactionId, - logVersion, - byteOffset, - checksum, - commitTimestamp - ); - } - @Override public void setCurrentLogVersion(long version) { logVersionRepository.setCurrentLogVersion(version); @@ -109,23 +74,67 @@ public long incrementAndGetCheckpointLogVersion() { } @Override - public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { - transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp); + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); } @Override public void setLastCommittedAndClosedTransactionId( - long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion ) { transactionIdStore.setLastCommittedAndClosedTransactionId( transactionId, checksum, commitTimestamp, + consensusIndex, byteOffset, logVersion ); } + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + @Override public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { } @@ -183,8 +192,4 @@ public Optional getDatabaseIdUuid(CursorContext cursorTracer) { public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { throw new IllegalStateException("Not supported"); } - - TransactionIdStore transactionIdStore() { - return this.transactionIdStore; - } } diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java index 465e50cba08..ef8e75b95f2 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java @@ -20,56 +20,86 @@ package org.neo4j.gds.compat._55; import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override public ClosedTransactionMetadata getLastClosedTransaction() { long[] metaData = this.closedTransactionId.get(); return new ClosedTransactionMetadata( metaData[0], new LogPosition(metaData[1], metaData[2]), (int) metaData[3], - metaData[4] + metaData[4], + metaData[5] ); } @Override - public void transactionClosed( + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( long transactionId, - long logVersion, - long byteOffset, int checksum, - long commitTimestamp + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion ) { - this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } @Override - public void resetLastClosedTransaction( + public void transactionClosed( long transactionId, long logVersion, long byteOffset, int checksum, - long commitTimestamp + long commitTimestamp, + long consensusIndex ) { - this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex}); } @Override - public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { - } - - @Override - public void setLastCommittedAndClosedTransactionId( - long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex ) { - } - - @Override - protected CursorContext getEmptyCursorContext() { - return CursorContext.NULL_CONTEXT; + this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex}); } } diff --git a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java index a07942bdb6d..589ae4775b3 100644 --- a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java +++ b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/internal/recordstorage/AbstractTransactionIdStore.java @@ -19,7 +19,6 @@ */ package org.neo4j.internal.recordstorage; -import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.storageengine.api.TransactionId; import org.neo4j.storageengine.api.TransactionIdStore; import org.neo4j.util.concurrent.ArrayQueueOutOfOrderSequence; @@ -29,12 +28,12 @@ import java.util.concurrent.atomic.AtomicReference; public abstract class AbstractTransactionIdStore implements TransactionIdStore { - private final AtomicLong committingTransactionId; + protected final AtomicLong committingTransactionId; protected final OutOfOrderSequence closedTransactionId; - private final AtomicReference committedTransactionId; - private final long previouslyCommittedTxId; - private final int initialTransactionChecksum; - private final long previouslyCommittedTxCommitTimestamp; + protected final AtomicReference committedTransactionId; + protected final long previouslyCommittedTxId; + protected final int initialTransactionChecksum; + protected final long previouslyCommittedTxCommitTimestamp; public AbstractTransactionIdStore() { this(1L, -559063315, 0L, 0L, 64L); @@ -48,94 +47,55 @@ public AbstractTransactionIdStore( long previouslyCommittedTxLogByteOffset ) { this.committingTransactionId = new AtomicLong(); - this.closedTransactionId = new ArrayQueueOutOfOrderSequence(-1L, 100, new long[1]); - this.committedTransactionId = new AtomicReference<>(new TransactionId(1L, -559063315, 0L)); + this.closedTransactionId = new ArrayQueueOutOfOrderSequence(-1L, 100, new long[5]); + this.committedTransactionId = new AtomicReference<>(transactionId(1L, -559063315, 0L)); assert previouslyCommittedTxId >= 1L : "cannot start from a tx id less than BASE_TX_ID"; - this.setLastCommittedAndClosedTransactionId( + initLastCommittedAndClosedTransactionId( previouslyCommittedTxId, checksum, previouslyCommittedTxCommitTimestamp, - previouslyCommittedTxLogByteOffset, previouslyCommittedTxLogVersion, - getEmptyCursorContext() + previouslyCommittedTxLogByteOffset ); this.previouslyCommittedTxId = previouslyCommittedTxId; this.initialTransactionChecksum = checksum; this.previouslyCommittedTxCommitTimestamp = previouslyCommittedTxCommitTimestamp; } - protected abstract CursorContext getEmptyCursorContext(); + protected abstract void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ); + protected abstract TransactionId transactionId(long transactionId, int checksum, long commitTimestamp); + + @Override public long nextCommittingTransactionId() { return this.committingTransactionId.incrementAndGet(); } + @Override public long committingTransactionId() { return this.committingTransactionId.get(); } - public synchronized void transactionCommitted( - long transactionId, - int checksum, - long commitTimestamp, - CursorContext cursorContext - ) { - TransactionId current = this.committedTransactionId.get(); - if (current == null || transactionId > current.transactionId()) { - this.committedTransactionId.set(new TransactionId(transactionId, checksum, commitTimestamp)); - } - - } - + @Override public long getLastCommittedTransactionId() { return this.committedTransactionId.get().transactionId(); } + @Override public TransactionId getLastCommittedTransaction() { return this.committedTransactionId.get(); } - public TransactionId getUpgradeTransaction() { - return new TransactionId( - this.previouslyCommittedTxId, - this.initialTransactionChecksum, - this.previouslyCommittedTxCommitTimestamp - ); - } - + @Override public long getLastClosedTransactionId() { return this.closedTransactionId.getHighestGapFreeNumber(); } - - public void setLastCommittedAndClosedTransactionId( - long transactionId, - int checksum, - long commitTimestamp, - long byteOffset, - long logVersion, - CursorContext cursorContext - ) { - this.committingTransactionId.set(transactionId); - this.committedTransactionId.set(new TransactionId(transactionId, checksum, commitTimestamp)); - this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); - } - - public void transactionClosed(long transactionId, long logVersion, long byteOffset, CursorContext cursorContext) { - this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset}); - } - - public void resetLastClosedTransaction( - long transactionId, - long byteOffset, - long logVersion, - boolean missingLogs, - CursorContext cursorContext - ) { - this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset}); - } - - public void flush(CursorContext cursorContext) { - } } From 06c83b6a320029a1a8deee920e2f3aa4642cf809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Tue, 21 Feb 2023 10:12:29 +0100 Subject: [PATCH 200/400] Fix string id collision bug in arrow create db --- .../java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java | 9 +++++---- .../java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java | 4 ++-- .../java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java | 4 ++-- .../java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java | 4 ++-- .../java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java | 4 ++-- .../java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java | 4 ++-- .../main/java/org/neo4j/gds/compat/Neo4jProxyApi.java | 3 ++- .../src/main/java/org/neo4j/gds/compat/Neo4jProxy.java | 5 +++-- 8 files changed, 20 insertions(+), 17 deletions(-) diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index b8d2acca3c4..a1fee3f9930 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -460,21 +460,22 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(Group.GLOBAL.id()); return new InputEntityIdVisitor.String() { @Override public void visitNodeId(InputEntityVisitor visitor, String id) { - visitor.id(id, Group.GLOBAL); + visitor.id(id, globalGroup); } @Override public void visitSourceId(InputEntityVisitor visitor, String id) { - visitor.startId(id, Group.GLOBAL); + visitor.startId(id, globalGroup); } @Override public void visitTargetId(InputEntityVisitor visitor, String id) { - visitor.endId(id, Group.GLOBAL); + visitor.endId(id, globalGroup); } }; } diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index 5709766e4d3..09cf149f75d 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -558,8 +558,8 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { - var globalGroup = new Group(0, null, null); + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); return new InputEntityIdVisitor.String() { @Override diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index a0d71680974..b9aaf948d57 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -556,8 +556,8 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { - var globalGroup = new Group(0, null, null); + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); return new InputEntityIdVisitor.String() { @Override diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index c1051e45511..d120fe4d9bd 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -556,8 +556,8 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { - var globalGroup = new Group(0, null, null); + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); return new InputEntityIdVisitor.String() { @Override diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index ef6f3408a53..112892e48e3 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -556,8 +556,8 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { - var globalGroup = new Group(0, null, null); + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); return new InputEntityIdVisitor.String() { @Override diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java index 2bb6641c6ea..53fea0a4e8c 100644 --- a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -557,8 +557,8 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } @Override - public InputEntityIdVisitor.String inputEntityStringIdVisitor() { - var globalGroup = new Group(0, null, null); + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); return new InputEntityIdVisitor.String() { @Override diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index ccd863bacae..7deab25edcf 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -39,6 +39,7 @@ import org.neo4j.internal.batchimport.input.Collector; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.ReadableGroups; import org.neo4j.internal.batchimport.staging.ExecutionMonitor; import org.neo4j.internal.helpers.HostnamePort; import org.neo4j.internal.id.IdGeneratorFactory; @@ -210,7 +211,7 @@ BatchImporter instantiateBatchImporter( InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType); - InputEntityIdVisitor.String inputEntityStringIdVisitor(); + InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups); Setting additionalJvm(); diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index ea25d7a6205..17ebe30a51e 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -39,6 +39,7 @@ import org.neo4j.internal.batchimport.input.Collector; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.ReadableGroups; import org.neo4j.internal.batchimport.staging.ExecutionMonitor; import org.neo4j.internal.helpers.HostnamePort; import org.neo4j.internal.id.IdGeneratorFactory; @@ -284,8 +285,8 @@ public static InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) return IMPL.inputEntityLongIdVisitor(idType); } - public static InputEntityIdVisitor.String inputEntityStringIdVisitor() { - return IMPL.inputEntityStringIdVisitor(); + public static InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + return IMPL.inputEntityStringIdVisitor(groups); } public static Input batchInputFrom(CompatInput compatInput) { From f61cbb8e730f7c601ac2e533b8c7931ba6879aac Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Tue, 21 Feb 2023 10:54:13 +0100 Subject: [PATCH 201/400] Untangle 5.5 and rc compat layers after cherry-pick --- .../_55/InMemoryMetaDataProviderImpl.java | 84 ++++++++----------- .../_55/InMemoryTransactionIdStoreImpl.java | 53 +++++------- 2 files changed, 57 insertions(+), 80 deletions(-) diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java index e551e1a75c1..ce7adaea00d 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java @@ -53,6 +53,40 @@ public ClosedTransactionMetadata getLastClosedTransaction() { return this.transactionIdStore.getLastClosedTransaction(); } + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp + ); + } + @Override public void setCurrentLogVersion(long version) { logVersionRepository.setCurrentLogVersion(version); @@ -74,67 +108,23 @@ public long incrementAndGetCheckpointLogVersion() { } @Override - public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { - transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp); } @Override public void setLastCommittedAndClosedTransactionId( - long transactionId, - int checksum, - long commitTimestamp, - long consensusIndex, - long byteOffset, - long logVersion + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion ) { transactionIdStore.setLastCommittedAndClosedTransactionId( transactionId, checksum, commitTimestamp, - consensusIndex, byteOffset, logVersion ); } - @Override - public void transactionClosed( - long transactionId, - long logVersion, - long byteOffset, - int checksum, - long commitTimestamp, - long consensusIndex - ) { - this.transactionIdStore.transactionClosed( - transactionId, - logVersion, - byteOffset, - checksum, - commitTimestamp, - consensusIndex - ); - } - - @Override - public void resetLastClosedTransaction( - long transactionId, - long logVersion, - long byteOffset, - int checksum, - long commitTimestamp, - long consensusIndex - ) { - this.transactionIdStore.resetLastClosedTransaction( - transactionId, - logVersion, - byteOffset, - checksum, - commitTimestamp, - consensusIndex - ); - } - @Override public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { } diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java index ef8e75b95f2..0a6f3e8c889 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryTransactionIdStoreImpl.java @@ -23,7 +23,6 @@ import org.neo4j.kernel.impl.transaction.log.LogPosition; import org.neo4j.storageengine.api.ClosedTransactionMetadata; import org.neo4j.storageengine.api.TransactionId; -import org.neo4j.storageengine.api.TransactionIdStore; public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { @@ -39,56 +38,30 @@ protected void initLastCommittedAndClosedTransactionId( previouslyCommittedTxId, checksum, previouslyCommittedTxCommitTimestamp, - TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, previouslyCommittedTxLogByteOffset, previouslyCommittedTxLogVersion ); } - @Override public ClosedTransactionMetadata getLastClosedTransaction() { long[] metaData = this.closedTransactionId.get(); return new ClosedTransactionMetadata( metaData[0], new LogPosition(metaData[1], metaData[2]), (int) metaData[3], - metaData[4], - metaData[5] + metaData[4] ); } - @Override - protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { - return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); - } - - @Override - public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { - - } - - @Override - public void setLastCommittedAndClosedTransactionId( - long transactionId, - int checksum, - long commitTimestamp, - long consensusIndex, - long byteOffset, - long logVersion - ) { - - } - @Override public void transactionClosed( long transactionId, long logVersion, long byteOffset, int checksum, - long commitTimestamp, - long consensusIndex + long commitTimestamp ) { - this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex}); + this.closedTransactionId.offer(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); } @Override @@ -97,9 +70,23 @@ public void resetLastClosedTransaction( long logVersion, long byteOffset, int checksum, - long commitTimestamp, - long consensusIndex + long commitTimestamp ) { - this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex}); + this.closedTransactionId.set(transactionId, new long[]{logVersion, byteOffset, checksum, commitTimestamp}); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp) { + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, int checksum, long commitTimestamp, long byteOffset, long logVersion + ) { + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp); } } From 92b3d087976bb9149805590049157c49338c015d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Tue, 21 Feb 2023 10:47:34 +0100 Subject: [PATCH 202/400] Apply group lookup also for int ids --- .../java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java | 10 ++++++---- .../org/neo4j/gds/compat/_51/Neo4jProxyImpl.java | 5 ++--- .../org/neo4j/gds/compat/_52/Neo4jProxyImpl.java | 5 ++--- .../org/neo4j/gds/compat/_53/Neo4jProxyImpl.java | 5 ++--- .../org/neo4j/gds/compat/_54/Neo4jProxyImpl.java | 5 ++--- .../org/neo4j/gds/compat/_55/Neo4jProxyImpl.java | 5 ++--- .../main/java/org/neo4j/gds/compat/Neo4jProxyApi.java | 2 +- .../src/main/java/org/neo4j/gds/compat/Neo4jProxy.java | 4 ++-- .../java/org/neo4j/gds/core/io/GraphStoreInput.java | 6 +++--- 9 files changed, 22 insertions(+), 25 deletions(-) diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index a1fee3f9930..73ffa9abab5 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -418,7 +418,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL: return new InputEntityIdVisitor.Long() { @@ -438,20 +438,22 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { } }; case INTEGER: + var globalGroup = groups.get(Group.GLOBAL.id()); + return new InputEntityIdVisitor.Long() { @Override public void visitNodeId(InputEntityVisitor visitor, long id) { - visitor.id(id, Group.GLOBAL); + visitor.id(id, globalGroup); } @Override public void visitSourceId(InputEntityVisitor visitor, long id) { - visitor.startId(id, Group.GLOBAL); + visitor.startId(id, globalGroup); } @Override public void visitTargetId(InputEntityVisitor visitor, long id) { - visitor.endId(id, Group.GLOBAL); + visitor.endId(id, globalGroup); } }; default: diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index 09cf149f75d..dd9564803aa 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -62,7 +62,6 @@ import org.neo4j.internal.batchimport.InputIterable; import org.neo4j.internal.batchimport.Monitor; import org.neo4j.internal.batchimport.input.Collector; -import org.neo4j.internal.batchimport.input.Group; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; import org.neo4j.internal.batchimport.input.InputEntityVisitor; @@ -513,7 +512,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL -> { return new InputEntityIdVisitor.Long() { @@ -534,7 +533,7 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { }; } case INTEGER -> { - var globalGroup = new Group(0, null, null); + var globalGroup = groups.get(null); return new InputEntityIdVisitor.Long() { @Override diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index b9aaf948d57..f7c51bffc1a 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -62,7 +62,6 @@ import org.neo4j.internal.batchimport.InputIterable; import org.neo4j.internal.batchimport.Monitor; import org.neo4j.internal.batchimport.input.Collector; -import org.neo4j.internal.batchimport.input.Group; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; import org.neo4j.internal.batchimport.input.InputEntityVisitor; @@ -511,7 +510,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL -> { return new InputEntityIdVisitor.Long() { @@ -532,7 +531,7 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { }; } case INTEGER -> { - var globalGroup = new Group(0, null, null); + var globalGroup = groups.get(null); return new InputEntityIdVisitor.Long() { @Override diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index d120fe4d9bd..f4b6072fe0a 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -62,7 +62,6 @@ import org.neo4j.internal.batchimport.InputIterable; import org.neo4j.internal.batchimport.Monitor; import org.neo4j.internal.batchimport.input.Collector; -import org.neo4j.internal.batchimport.input.Group; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; import org.neo4j.internal.batchimport.input.InputEntityVisitor; @@ -511,7 +510,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL -> { return new InputEntityIdVisitor.Long() { @@ -532,7 +531,7 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { }; } case INTEGER -> { - var globalGroup = new Group(0, null, null); + var globalGroup = groups.get(null); return new InputEntityIdVisitor.Long() { @Override diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index 112892e48e3..8f97bac8b98 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -62,7 +62,6 @@ import org.neo4j.internal.batchimport.InputIterable; import org.neo4j.internal.batchimport.Monitor; import org.neo4j.internal.batchimport.input.Collector; -import org.neo4j.internal.batchimport.input.Group; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; import org.neo4j.internal.batchimport.input.InputEntityVisitor; @@ -511,7 +510,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL -> { return new InputEntityIdVisitor.Long() { @@ -532,7 +531,7 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { }; } case INTEGER -> { - var globalGroup = new Group(0, null, null); + var globalGroup = groups.get(null); return new InputEntityIdVisitor.Long() { @Override diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java index 53fea0a4e8c..16a57839d6c 100644 --- a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -62,7 +62,6 @@ import org.neo4j.internal.batchimport.InputIterable; import org.neo4j.internal.batchimport.Monitor; import org.neo4j.internal.batchimport.input.Collector; -import org.neo4j.internal.batchimport.input.Group; import org.neo4j.internal.batchimport.input.IdType; import org.neo4j.internal.batchimport.input.Input; import org.neo4j.internal.batchimport.input.InputEntityVisitor; @@ -512,7 +511,7 @@ public Input batchInputFrom(CompatInput compatInput) { } @Override - public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { switch (idType) { case ACTUAL -> { return new InputEntityIdVisitor.Long() { @@ -533,7 +532,7 @@ public void visitTargetId(InputEntityVisitor visitor, long id) { }; } case INTEGER -> { - var globalGroup = new Group(0, null, null); + var globalGroup = groups.get(null); return new InputEntityIdVisitor.Long() { @Override diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index 7deab25edcf..ed0fedb0173 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -209,7 +209,7 @@ BatchImporter instantiateBatchImporter( Input batchInputFrom(CompatInput compatInput); - InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType); + InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups); InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups); diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index 17ebe30a51e..c319f814b1c 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -281,8 +281,8 @@ public static BatchImporter instantiateBatchImporter( ); } - public static InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType) { - return IMPL.inputEntityLongIdVisitor(idType); + public static InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + return IMPL.inputEntityLongIdVisitor(idType, groups); } public static InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java index 0384079262b..7a74e29dbc8 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/GraphStoreInput.java @@ -73,18 +73,18 @@ enum IdMode implements Supplier { MAPPING(IdType.INTEGER, new Groups()) { @Override public InputEntityIdVisitor.Long get() { - return Neo4jProxy.inputEntityLongIdVisitor(IdType.INTEGER); + return Neo4jProxy.inputEntityLongIdVisitor(IdType.INTEGER, readableGroups); } }, ACTUAL(IdType.ACTUAL, Groups.EMPTY) { @Override public InputEntityIdVisitor.Long get() { - return Neo4jProxy.inputEntityLongIdVisitor(IdType.ACTUAL); + return Neo4jProxy.inputEntityLongIdVisitor(IdType.ACTUAL, readableGroups); } }; private final IdType idType; - private final ReadableGroups readableGroups; + final ReadableGroups readableGroups; IdMode(IdType idType, ReadableGroups readableGroups) { this.idType = idType; From a9c00bc1cfd224ae38dbc346109f6bb52e0d3d5c Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 22 Feb 2023 11:27:26 +0100 Subject: [PATCH 203/400] Eliminate race when mapping intermediate ids We have a TOCTOU situation in the LazyIdMapBuilder. We check if a node is already mapped and if not, we add it to the nodesBuilder. This also works as a deduplication mechanism. When we add the same node from multiple threads, those two threads can both observe the node to be missing, and both will add the node. This will overwrite the intermediate mapping for that node for one thread, but we will insert the intermediate node ids from both threads into the final id map. The missing mapping leads to a sparse HLA for the originalId mapping, which we never expect to be sparse. This leads to an originalId of 0 for those lost nodes and this can result in all kinds of errors, e.g. a NOT_FOUND later on. Co-Authored-By: Martin Junghanns --- .../gds/core/loading/LazyIdMapBuilder.java | 15 ++++----- .../core/utils/paged/ShardedLongLongMap.java | 24 +++++++------- .../utils/paged/ShardedLongLongMapTest.java | 33 +++++++++++++++++++ 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index 4587dee0ccd..fbabbc20506 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -55,15 +55,13 @@ public void prepareForFlush() { } public long addNode(long nodeId, NodeLabelToken nodeLabels) { - var intermediateId = this.intermediateIdMapBuilder.toMappedNodeId(nodeId); + long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); // deduplication - if (intermediateId != IdMap.NOT_FOUND) { - return intermediateId; + if (intermediateId < 0) { + return -(intermediateId + 1); } - intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); - this.nodesBuilder.addNode(intermediateId, nodeLabels); return intermediateId; @@ -74,14 +72,13 @@ public long addNodeWithProperties( PropertyValues properties, NodeLabelToken nodeLabels ) { - var intermediateId = this.intermediateIdMapBuilder.toMappedNodeId(nodeId); + long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); // deduplication - if (intermediateId != IdMap.NOT_FOUND) { - return intermediateId; + if (intermediateId < 0) { + return -(intermediateId + 1); } - intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); if (properties.isEmpty()) { this.nodesBuilder.addNode(intermediateId, nodeLabels); } else { diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java index 1a2d88609f5..94a4b897586 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java @@ -213,6 +213,11 @@ public static final class Builder { .toArray(Shard[]::new); } + /** + * Add a node to the mapping. + * @return {@code mappedId >= 0} if the node was added, + * or {@code -(mappedId) - 1} if the node was already mapped. + */ public long addNode(long nodeId) { var shard = findShard(nodeId, this.shards, this.shardShift, this.shardMask); try (var ignoredLock = shard.acquireLock()) { @@ -220,11 +225,6 @@ public long addNode(long nodeId) { } } - public long toMappedNodeId(long nodeId) { - var shard = findShard(nodeId, this.shards, this.shardShift, this.shardMask); - return shard.toMappedNodeId(nodeId); - } - public ShardedLongLongMap build() { return ShardedLongLongMap.build( this.nodeCount.get(), @@ -252,15 +252,15 @@ private Shard(AtomicLong nextId) { this.nextId = nextId; } - long toMappedNodeId(long nodeId) { - return mapping.getIfAbsent(nodeId, IdMap.NOT_FOUND); - } - long addNode(long nodeId) { this.assertIsUnderLock(); - long internalId = this.nextId.getAndIncrement(); - mapping.put(nodeId, internalId); - return internalId; + long mappedId = mapping.getIfAbsent(nodeId, -1); + if (mappedId != IdMap.NOT_FOUND) { + return -mappedId - 1; + } + mappedId = nextId.getAndIncrement(); + mapping.put(nodeId, mappedId); + return mappedId; } } } diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java index ecebd86ad05..2393f1a7a27 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMapTest.java @@ -110,6 +110,16 @@ void testContains(@ForAll("ids") long[] originalIds) { .forEach(id -> assertThat(map.contains(id)).isFalse()); } + @Property + void testAddNode(@ForAll("ids") long[] originalIds) { + var builder = builder(1); + for (int i = 0; i < originalIds.length; i++) { + long originalId = originalIds[i]; + long mappedId = builder.addNode(originalId); + assertThat(mappedId).isEqualTo(i); + } + } + @Property void testMaxOriginalId(@ForAll("ids") long[] originalIds) { var builder = builder(1); @@ -189,6 +199,8 @@ void testAddingMultipleNodesInParallel(@ForAll("fixedSizeIds") long[] originalId abstract TestBuilder builder(int concurrency); interface TestBuilder { + long addNode(long nodeId); + void addNodes(long... nodeIds); ShardedLongLongMap build(); @@ -196,6 +208,16 @@ interface TestBuilder { static class DefaultBuilderTest extends ShardedLongLongMapTest { + @Property + void testAddNodeWithDuplicates(@ForAll("ids") long[] originalIds) { + var builder = builder(1); + for (long originalId : originalIds) { + long mappedId = builder.addNode(originalId); + long duplicateMappedId = builder.addNode(originalId); + assertThat(duplicateMappedId).isEqualTo(-mappedId - 1); + } + } + @Override TestBuilder builder(int concurrency) { return new DefaultBuilder(concurrency); @@ -208,6 +230,11 @@ private static final class DefaultBuilder implements TestBuilder { this.inner = ShardedLongLongMap.builder(concurrency); } + @Override + public long addNode(long nodeId) { + return inner.addNode(nodeId); + } + @Override public void addNodes(long... nodeIds) { for (long nodeId : nodeIds) { @@ -248,6 +275,12 @@ private static final class BatchedBuilder implements TestBuilder { inner = ShardedLongLongMap.batchedBuilder(concurrency); } + @Override + public long addNode(long nodeId) { + var batch = inner.prepareBatch(1); + return batch.addNode(nodeId); + } + @Override public void addNodes(long... nodeIds) { var batch = inner.prepareBatch(nodeIds.length); From 7f9b479089eba77558881e0068e99fbffb2ded2b Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 22 Feb 2023 11:49:16 +0100 Subject: [PATCH 204/400] Add test to reproduce race for duplicate original ids Co-Authored-By: Paul Horn --- .../core/loading/LazyIdMapBuilderTest.java | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java diff --git a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java new file mode 100644 index 00000000000..4924b74beb5 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.core.concurrency.ParallelUtil; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.loading.construction.NodeLabelTokens; +import org.neo4j.gds.core.utils.partition.PartitionUtils; + +import java.util.Collections; +import java.util.Optional; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +class LazyIdMapBuilderTest { + + @Test + void parallelAddDuplicateNodes() { + int concurrency = 8; + long idCount = 100_000; + var rng = new Random(42); + + var lazyIdMapBuilder = new LazyIdMapBuilder(concurrency, false, false); + + var idList = LongStream.rangeClosed(1, idCount) + .flatMap(id -> rng.nextBoolean() ? LongStream.of(id, id) : LongStream.of(id)) + .limit(idCount) + .boxed() + .collect(Collectors.toList()); + + // shuffle to spread duplicates across the whole range + Collections.shuffle(idList); + + var idArray = idList.stream().mapToLong(l -> l).toArray(); + + var tasks = PartitionUtils.rangePartition(concurrency, idCount, partition -> (Runnable) () -> { + int start = (int) partition.startNode(); + int end = (int) (start + partition.nodeCount()); + for (int i = start; i < end ; i++) { + long originalId = idArray[i]; + // We potentially insert the same original id from multiple threads. + // This MUST not lead to new intermediate ids generated internally. + lazyIdMapBuilder.addNode(originalId, NodeLabelTokens.empty()); + } + + }, Optional.empty()); + + ParallelUtil.run(tasks, Pools.DEFAULT); + + var highLimitIdMap = lazyIdMapBuilder.build().idMap(); + + Assertions.assertThat(highLimitIdMap.nodeCount()).isLessThan(idCount); + + for (int internalNodeId = 0; internalNodeId < highLimitIdMap.nodeCount(); internalNodeId++) { + Assertions + .assertThat(highLimitIdMap.toOriginalNodeId(internalNodeId)) + .as("Internal node id %s is not mapped", internalNodeId) + .isNotEqualTo(0); + } + } + +} From 6a529b5da17df613d400a2f917aad466b192fcc1 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 22 Feb 2023 11:59:27 +0100 Subject: [PATCH 205/400] Replace -1 with a constant Co-Authored-By: Martin Junghanns --- .../java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java index 94a4b897586..62c0f10ec22 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/ShardedLongLongMap.java @@ -254,7 +254,7 @@ private Shard(AtomicLong nextId) { long addNode(long nodeId) { this.assertIsUnderLock(); - long mappedId = mapping.getIfAbsent(nodeId, -1); + long mappedId = mapping.getIfAbsent(nodeId, IdMap.NOT_FOUND); if (mappedId != IdMap.NOT_FOUND) { return -mappedId - 1; } From 161d37b83ac9f6d02e41a539c039d78c588dabbb Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 22 Feb 2023 13:59:01 +0100 Subject: [PATCH 206/400] Remove unused import Co-Authored-By: Martin Junghanns --- .../main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index fbabbc20506..14ba2ca6124 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -20,7 +20,6 @@ package org.neo4j.gds.core.loading; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.api.IdMap; import org.neo4j.gds.api.PartialIdMap; import org.neo4j.gds.api.properties.nodes.NodePropertyStore; import org.neo4j.gds.api.schema.MutableNodeSchema; From 46ee937bc13f40e807e915b842e57496badc7d82 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 22 Feb 2023 14:55:42 +0100 Subject: [PATCH 207/400] It is absolutely crucial that a semicolon is not preceded by whitespace Co-Authored-By: Martin Junghanns --- .../java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java index 4924b74beb5..947f4846c3e 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/LazyIdMapBuilderTest.java @@ -56,7 +56,7 @@ void parallelAddDuplicateNodes() { var tasks = PartitionUtils.rangePartition(concurrency, idCount, partition -> (Runnable) () -> { int start = (int) partition.startNode(); int end = (int) (start + partition.nodeCount()); - for (int i = start; i < end ; i++) { + for (int i = start; i < end; i++) { long originalId = idArray[i]; // We potentially insert the same original id from multiple threads. // This MUST not lead to new intermediate ids generated internally. From 848f91b972500aac2658eeb951e18ffafbefc46f Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 22 Feb 2023 16:41:55 +0100 Subject: [PATCH 208/400] Remove cypher deprecation warnings before comparing results --- .../java/org/neo4j/gds/AllShortestPathsDocTest.java | 5 ++--- test-utils/src/main/java/org/neo4j/gds/BaseTest.java | 10 ++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java index fa3fa7c61c5..8762f8bb14d 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java @@ -24,7 +24,6 @@ import org.neo4j.gds.catalog.GraphProjectProc; import org.neo4j.gds.functions.IsFiniteFunc; import org.neo4j.gds.shortestpaths.AllShortestPathsProc; -import org.neo4j.graphdb.Result; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -83,7 +82,7 @@ void shouldStream() { " ORDER BY distance DESC, source ASC, target ASC" + " LIMIT 10"; - String actual = runQuery(query, Result::resultAsString); + String actual = runQuery(query, BaseTest::resultAsStringNoDeprecation); String expected = "+----------------------------+" + NL + "| source | target | distance |" + NL + "+----------------------------+" + NL + @@ -127,7 +126,7 @@ void shouldStreamWithCypherProjection() { " ORDER BY distance DESC, source ASC, target ASC" + " LIMIT 10"; - String actual = runQuery(query, Result::resultAsString); + String actual = runQuery(query, BaseTest::resultAsStringNoDeprecation); String expected = "+----------------------------+" + NL + "| source | target | distance |" + NL + "+----------------------------+" + NL + diff --git a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java index e569c861835..de165897dbf 100644 --- a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java @@ -42,6 +42,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import java.util.regex.Pattern; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -262,6 +263,15 @@ protected void assertCypherResult( TestSupport.assertCypherResult(db, query, queryParameters, expected); } + private static final Pattern DEPRECATION_NOTE = Pattern.compile( + "The query used a deprecated function.+", + Pattern.DOTALL + ); + + public static String resultAsStringNoDeprecation(Result result) { + return DEPRECATION_NOTE.matcher(result.resultAsString()).replaceAll(""); + } + static { // configure AssertJ to consider 'name()' as a method for the property 'name' instead of just 'getName()' Assertions.setExtractBareNamePropertyMethods(true); From be473e7f440ef8c767c4bedf8b7b33bacc8ce6c0 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Thu, 23 Feb 2023 10:12:22 +0100 Subject: [PATCH 209/400] Move test specific result stripping to the test that uses it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Jonatan Jäderberg --- .../org/neo4j/gds/AllShortestPathsDocTest.java | 16 ++++++++++++++-- .../src/main/java/org/neo4j/gds/BaseTest.java | 10 ---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java index 8762f8bb14d..197a08375ad 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java @@ -24,6 +24,9 @@ import org.neo4j.gds.catalog.GraphProjectProc; import org.neo4j.gds.functions.IsFiniteFunc; import org.neo4j.gds.shortestpaths.AllShortestPathsProc; +import org.neo4j.graphdb.Result; + +import java.util.regex.Pattern; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -82,7 +85,7 @@ void shouldStream() { " ORDER BY distance DESC, source ASC, target ASC" + " LIMIT 10"; - String actual = runQuery(query, BaseTest::resultAsStringNoDeprecation); + String actual = runQuery(query, AllShortestPathsDocTest::resultAsStringNoDeprecation); String expected = "+----------------------------+" + NL + "| source | target | distance |" + NL + "+----------------------------+" + NL + @@ -126,7 +129,7 @@ void shouldStreamWithCypherProjection() { " ORDER BY distance DESC, source ASC, target ASC" + " LIMIT 10"; - String actual = runQuery(query, BaseTest::resultAsStringNoDeprecation); + String actual = runQuery(query, AllShortestPathsDocTest::resultAsStringNoDeprecation); String expected = "+----------------------------+" + NL + "| source | target | distance |" + NL + "+----------------------------+" + NL + @@ -145,4 +148,13 @@ void shouldStreamWithCypherProjection() { assertEquals(expected, actual); } + + private static final Pattern DEPRECATION_NOTE = Pattern.compile( + "The query used a deprecated function.+", + Pattern.DOTALL + ); + + private static String resultAsStringNoDeprecation(Result result) { + return DEPRECATION_NOTE.matcher(result.resultAsString()).replaceAll(""); + } } diff --git a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java index de165897dbf..e569c861835 100644 --- a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java @@ -42,7 +42,6 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; -import java.util.regex.Pattern; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -263,15 +262,6 @@ protected void assertCypherResult( TestSupport.assertCypherResult(db, query, queryParameters, expected); } - private static final Pattern DEPRECATION_NOTE = Pattern.compile( - "The query used a deprecated function.+", - Pattern.DOTALL - ); - - public static String resultAsStringNoDeprecation(Result result) { - return DEPRECATION_NOTE.matcher(result.resultAsString()).replaceAll(""); - } - static { // configure AssertJ to consider 'name()' as a method for the property 'name' instead of just 'getName()' Assertions.setExtractBareNamePropertyMethods(true); From a08caa53c9d2a88b3ad7a3875a3e23aa930bd1f9 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Wed, 22 Feb 2023 15:49:07 +0100 Subject: [PATCH 210/400] Only make Storage Engines available that can actually run on this Neo4j --- .../compat/_44/InMemoryStorageEngineFactory.java | 6 ++++++ .../compat/_51/InMemoryStorageEngineFactory.java | 6 ++++++ .../compat/_52/InMemoryStorageEngineFactory.java | 6 ++++++ .../compat/_53/InMemoryStorageEngineFactory.java | 6 ++++++ .../compat/_54/InMemoryStorageEngineFactory.java | 6 ++++++ .../compat/_55/InMemoryStorageEngineFactory.java | 6 ++++++ .../neo4j/gds/compat/StorageEngineProxyApi.java | 14 ++++++++++++++ 7 files changed, 50 insertions(+) diff --git a/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java index 7ac94920456..ef689ef60ab 100644 --- a/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java +++ b/compatibility/4.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_44/InMemoryStorageEngineFactory.java @@ -22,6 +22,8 @@ import org.neo4j.annotations.service.ServiceProvider; import org.neo4j.configuration.Config; import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.id.IdController; import org.neo4j.internal.id.IdGeneratorFactory; @@ -73,6 +75,10 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-44"; + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_4_4, StorageEngineFactory.class); + } + private final InMemoryMetaDataProviderImpl metadataProvider = new InMemoryMetaDataProviderImpl(); @Override diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java index bdbf2a266d4..14dd53ef3fe 100644 --- a/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java @@ -27,6 +27,8 @@ import org.neo4j.consistency.report.ConsistencySummaryStatistics; import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.batchimport.AdditionalInitialIds; import org.neo4j.internal.batchimport.BatchImporter; @@ -113,6 +115,10 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-51"; + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_1, StorageEngineFactory.class); + } + private final InMemoryMetaDataProviderImpl metadataProvider = new InMemoryMetaDataProviderImpl(); @Override diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java index 4c3fa12ad27..04a977b6d07 100644 --- a/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java @@ -27,6 +27,8 @@ import org.neo4j.consistency.report.ConsistencySummaryStatistics; import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.batchimport.AdditionalInitialIds; import org.neo4j.internal.batchimport.BatchImporter; @@ -113,6 +115,10 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-52"; + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_2, StorageEngineFactory.class); + } + private final InMemoryMetaDataProviderImpl metadataProvider = new InMemoryMetaDataProviderImpl(); @Override diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java index 696ee516c8c..90d66499995 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -28,6 +28,8 @@ import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; import org.neo4j.function.ThrowingSupplier; import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.batchimport.AdditionalInitialIds; import org.neo4j.internal.batchimport.BatchImporter; @@ -112,6 +114,10 @@ @ServiceProvider public class InMemoryStorageEngineFactory implements StorageEngineFactory { + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_3, StorageEngineFactory.class); + } + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-53"; @Override diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java index 6b66f4e3153..5bcba1cb8a7 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -28,6 +28,8 @@ import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; import org.neo4j.function.ThrowingSupplier; import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.batchimport.AdditionalInitialIds; import org.neo4j.internal.batchimport.BatchImporter; @@ -116,6 +118,10 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-54"; + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_4, StorageEngineFactory.class); + } + // Record storage = 0, Freki = 1 // Let's leave some room for future storage engines // This arbitrary seems quite future-proof diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java index 7cfe623636e..c8c0218f65c 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -28,6 +28,8 @@ import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; import org.neo4j.function.ThrowingSupplier; import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.internal.batchimport.AdditionalInitialIds; import org.neo4j.internal.batchimport.BatchImporter; @@ -113,6 +115,10 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-55"; + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_5, StorageEngineFactory.class); + } + // Record storage = 0, Freki = 1 // Let's leave some room for future storage engines // This arbitrary seems quite future-proof diff --git a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java index 8245277e0c2..813163c80e9 100644 --- a/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java +++ b/compatibility/api/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/StorageEngineProxyApi.java @@ -35,8 +35,22 @@ import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; import org.neo4j.token.TokenHolders; +import java.util.Locale; + public interface StorageEngineProxyApi { + static void requireNeo4jVersion(Neo4jVersion version, Class self) { + if (Neo4jVersion.findNeo4jVersion() != version) { + throw new IllegalStateException(String.format( + Locale.ENGLISH, + "This '%s' instance is only compatible with Neo4j version %s, but Neo4j version %s has been detected.", + self.getName(), + version, + Neo4jVersion.findNeo4jVersion() + )); + } + } + // InMemoryStorageEngineBuilder inMemoryStorageEngineBuilder( // DatabaseLayout databaseLayout, // TokenHolders tokenHolders From 83b72d5882d22be579b74509ed2925f8e1d82380 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 23 Feb 2023 13:21:07 +0100 Subject: [PATCH 211/400] 2.3.0 mention --- .../ROOT/partials/algorithms/leiden/specific-configuration.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/modules/ROOT/partials/algorithms/leiden/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/leiden/specific-configuration.adoc index 6bc900c22f6..ca18c820a4b 100644 --- a/doc/modules/ROOT/partials/algorithms/leiden/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/leiden/specific-configuration.adoc @@ -5,3 +5,4 @@ | xref:common-usage/running-algos.adoc#common-configuration-tolerance[tolerance] | Float | 0.0001 | yes | Minimum change in modularity between iterations. If the modularity changes less than the tolerance value, the result is considered stable and the algorithm returns. | includeIntermediateCommunities | Boolean | false | yes | Indicates whether to write intermediate communities. If set to false, only the final community is persisted. | xref:common-usage/running-algos.adoc#common-configuration-seed-property[seedProperty] | String | n/a | yes | Used to set the initial community for a node. The property value needs to be a non-negative number. +| consecutiveIds | Boolean | false | yes | Flag to decide whether component identifiers are mapped into a consecutive id space (requires additional memory). Cannot be used in combination with the `includeIntermediateCommunities` flag. From ada911c6745dee6b25763439210c6407bb35148c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 23 Feb 2023 14:55:55 +0100 Subject: [PATCH 212/400] Fix estimation for smallest graph Also making sure that the EstimatonCLI will work in the future with very small graphs. --- .../gds/leiden/GraphAggregationPhase.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java b/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java index 14c90a3ac52..19a08ab1ad9 100644 --- a/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java +++ b/algo/src/main/java/org/neo4j/gds/leiden/GraphAggregationPhase.java @@ -53,22 +53,22 @@ class GraphAggregationPhase { static MemoryEstimation memoryEstimation() { return MemoryEstimations.builder(GraphAggregationPhase.class) - .rangePerGraphDimension("aggregated graph", (rootGraphDimensions, concurrency) -> { + .rangePerGraphDimension("aggregated graph", (rootDimensions, concurrency) -> { // The input graph might have multiple node and relationship properties // but the aggregated graph will never have more than a single relationship property // so let's - var maxDimensions = ImmutableGraphDimensions - .builder() - .from(rootGraphDimensions) - .build(); + + // Handle the case where the input graph has only one node + var minNodeCount = Math.min(2, rootDimensions.nodeCount()); + var minRelCount = Math.min(1, rootDimensions.relCountUpperBound()); var minDimensions = ImmutableGraphDimensions .builder() - .nodeCount(2) - .highestPossibleNodeCount(2) - .relationshipCounts(Map.of(RelationshipType.of("foo"), 1L)) - .relCountUpperBound(1) - .highestRelationshipId(1) + .nodeCount(minNodeCount) + .highestPossibleNodeCount(minNodeCount) + .relationshipCounts(Map.of(RelationshipType.of("foo"), minRelCount)) + .relCountUpperBound(minRelCount) + .highestRelationshipId(minRelCount) .build(); var relationshipProjections = ImmutableRelationshipProjections.builder() @@ -89,7 +89,7 @@ static MemoryEstimation memoryEstimation() { false ); var min = memoryEstimation.estimate(minDimensions, concurrency).memoryUsage().min; - var max = memoryEstimation.estimate(maxDimensions, concurrency).memoryUsage().max; + var max = memoryEstimation.estimate(rootDimensions, concurrency).memoryUsage().max; return MemoryRange.of(min, max); }).perNode("sorted communities", HugeLongArray::memoryEstimation) From e074af39f8b5c969ddb79917ab6bd63bde2b4e74 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 23 Feb 2023 15:31:23 +0100 Subject: [PATCH 213/400] increae time limit --- .../gds/similarity/nodesim/NodeSimilarityTerminationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java index 079603b904c..a07f3c10711 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTerminationTest.java @@ -65,7 +65,7 @@ void shouldTerminate() { db, nodeSimilarity, nhs -> nodeSimilarity.computeToStream(), - 100 + 200 ); } From 33dabb9ef9916ca17a20f3e6321916ef0ab1c1a6 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 24 Feb 2023 06:29:44 +0000 Subject: [PATCH 214/400] Use tables in config settings production page --- .../configuration-settings.adoc | 198 ++++++++++-------- 1 file changed, 112 insertions(+), 86 deletions(-) diff --git a/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc index 0eccf5a1676..e59453cb88a 100644 --- a/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc +++ b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc @@ -10,106 +10,132 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [.enterprise-edition] == GDS and Arrow -`gds.arrow.abortion_timeout` - -* The maximum time to wait for the next command before aborting the import process. -* defaultValue: `10m` -* validValues: a duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`) -* dynamic: `false` - -`gds.arrow.advertised_listen_address` - -* Address that clients should use to connect to the GDS Arrow Flight Server. -* defaultValue: `:8491` -* validValues: a socket address in the format `hostname:port`, `hostname` or `:port`. If missing port or hostname it is acquired from `gds.arrow.listen_address` -* dynamic: `false` - -`gds.arrow.batch_size` - -* The batch size used for arrow property export. -* defaultValue: `10000` -* validValues: an integer -* dynamic: `true` - -`gds.arrow.enabled` - -* Enable the GDS Arrow Flight Server. -* defaultValue: `false` -* validValues: a boolean -* dynamic: `false` - -`gds.arrow.encryption.never` - -* Never activate server-side encryption for the GDS Arrow Flight Server. -* defaultValue: `false` -* validValues: a boolean -* dynamic: `false` - -`gds.arrow.listen_address` - -* The maximum time to wait for the next command before aborting the import process. -* defaultValue: `10m` -* validValues: a duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`) -* dynamic: `false` +.gds.arrow.abortion_timeout +[cols="1,4", caption =] +|=== +| Description | The maximum time to wait for the next command before aborting the import process. +| Default Value | `10m` +| Valid Values | A duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`). +| Dynamic | `false` +|=== + +.gds.arrow.advertised_listen_address +[cols="1,4", caption =] +|=== +| Description | Address that clients should use to connect to the GDS Arrow Flight Server. +| Default Value | `:8491` +| Valid Values | A socket address in the format `hostname:port`, `hostname` or `:port`. If missing port or hostname it is acquired from `gds.arrow.listen_address`. +| Dynamic | `false` +|=== + +.gds.arrow.batch_size +[cols="1,4", caption =] +|=== +| Description | The batch size used for arrow property export. +| Default Value | `10000` +| Valid Values | An integer. +| Dynamic | `true` +|=== + +.gds.arrow.enabled +[cols="1,4", caption =] +|=== +| Description | Enable the GDS Arrow Flight Server. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + +.gds.arrow.encryption.never +[cols="1,4", caption =] +|=== +| Description | Never activate server-side encryption for the GDS Arrow Flight Server. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + +.gds.arrow.listen_address +[cols="1,4", caption =] +|=== +| Description | Address the GDS Arrow Flight Server should bind to. +| Default Value | `localhost:8491` +| Valid Values | A socket address in the format `hostname:port`, `hostname` or `:port`. +| Dynamic | `false` +|=== [.enterprise-edition] == Neo4j Cluster -`gds.cluster.tx.max.size` - -* Set the maximum transaction size for GDS write back when running in Neo4j Cluster. -* defaultValue: `100000` -* validValues: an integer, must be set greater than or equal to the value of `gds.cluster.tx.min.size` -* dynamic: `false` - -`gds.cluster.tx.min.size` - -* Set the minimum transaction size for GDS write back when running in Neo4j Cluster. -* defaultValue: `10000` -* validValues: an integer -* dynamic: `false` +.gds.cluster.tx.max.size +[cols="1,4", caption =] +|=== +| Description | Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +| Default Value | `100000` +| Valid Values | An integer, must be set greater than or equal to the value of `gds.cluster.tx.min.size`. +| Dynamic | `false` +|=== + +.gds.cluster.tx.min.size +[cols="1,4", caption =] +|=== +| Description | Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +| Default Value | `10000` +| Valid Values | An integer. +| Dynamic | `false` +|=== == GDS Enterprise Edition -`gds.enterprise.license_file` - -* Sets the location of the file that contains the Neo4j Graph Data Science library license key. -* defaultValue: `No Value` -* validValues: an absolute path -* dynamic: `false` +.gds.enterprise.license_file +[cols="1,4", caption =] +|=== +| Description | Sets the location of the file that contains the Neo4j Graph Data Science library license key. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== == GDS Export -`gds.export.location` - -* Sets the export location for file based exports. -* defaultValue: `No Value` -* validValues: an absolute path -* dynamic: `false` - -`gds.model.store_location` label:enterprise-edition[Enterprise Edition] - -* Sets the location where persisted models are stored. -* defaultValue: `No Value` -* validValues: an absolute path -* dynamic: `false` +.gds.export.location +[cols="1,4", caption = ] +|=== +| Description | Sets the export location for file based exports. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== + +.gds.model.store_location label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Sets the location where persisted models are stored. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== == Miscellaneous -`gds.progress_tracking_enabled` - -* Enable progress logging tracking. -* defaultValue: `true` -* validValues: a boolean -* dynamic: `false` - -`gds.validate_using_max_memory_estimation` - -* Use maximum memory estimation in procedure memory guard. -* defaultValue: `false` -* validValues: a boolean -* dynamic: `false` +.gds.progress_tracking_enabled +[cols="1,4", caption =] +|=== +| Description | Enable progress logging tracking. +| Default Value | `true` +| Valid Values | A boolean. +| Dynamic | `false` +|=== + +.gds.validate_using_max_memory_estimation +[cols="1,4", caption =] +|=== +| Description | Use maximum memory estimation in procedure memory guard. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== From 5f2427e618c1e60f4c0551ec1423e5918d47935b Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 24 Feb 2023 09:41:09 +0000 Subject: [PATCH 215/400] Use tables in config settings appendix page --- .../configuration-settings.adoc | 217 ++++++++++-------- 1 file changed, 121 insertions(+), 96 deletions(-) diff --git a/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc index f90c28614d6..a3e02be7c91 100644 --- a/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc +++ b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc @@ -6,135 +6,160 @@ This page describes the available configuration settings in GDS. Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4j-conf/#neo4j-conf[neo4j.conf] file for details on how to use configuration settings. -All settings - -* <>: label:enterprise-edition[Enterprise Edition] The maximum time to wait for the next command before aborting the import process. -* <>: label:enterprise-edition[Enterprise Edition] Address that clients should use to connect to the GDS Arrow Flight Server. -* <>: label:enterprise-edition[Enterprise Edition] The batch size used for arrow property export. -* <>: label:enterprise-edition[Enterprise Edition] Enable the GDS Arrow Flight Server. -* <>: label:enterprise-edition[Enterprise Edition] Never activate server-side encryption for the GDS Arrow Flight Server. -* <>: label:enterprise-edition[Enterprise Edition] Address the GDS Arrow Flight Server should bind to. -* <>: label:enterprise-edition[Enterprise Edition] Set the maximum transaction size for GDS on Read Replica -* <>: label:enterprise-edition[Enterprise Edition] Set the minimum transaction size for GDS on Read Replica -* <>: Sets the location of the file that contains the Neo4j Graph Data Science library license key -* <>: Sets the export location for file based exports. -* <>: label:enterprise-edition[Enterprise Edition] Sets the location where persisted models are stored. -* <>: Enable progress logging tracking. -* <>: Use maximum memory estimation in procedure memory guard. - +.All settings +[cols="2,1,2", caption =] +|=== +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | The maximum time to wait for the next command before aborting the import process. +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Address that clients should use to connect to the GDS Arrow Flight Server. +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | The batch size used for arrow property export. +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Enable the GDS Arrow Flight Server. +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Never activate server-side encryption for the GDS Arrow Flight Server. +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Address the GDS Arrow Flight Server should bind to. +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +<.^| <> | | Sets the location of the file that contains the Neo4j Graph Data Science library license key +<.^| <> | | Sets the export location for file based exports. +<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Sets the location where persisted models are stored. +<.^| <> | | Enable progress logging tracking. +<.^| <> | | Use maximum memory estimation in procedure memory guard. +|=== [[gds.arrow.abortion_timeout]] -`gds.arrow.abortion_timeout` label:enterprise-edition[Enterprise Edition] - -* The maximum time to wait for the next command before aborting the import process. -* defaultValue: `10m` -* validValues: a duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`) -* dynamic: `false` - +.gds.arrow.abortion_timeout label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | The maximum time to wait for the next command before aborting the import process. +| Default Value | `10m` +| Valid Values | A duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`). +| Dynamic | `false` +|=== [[gds.arrow.advertised_listen_address]] -`gds.arrow.advertised_listen_address` label:enterprise-edition[Enterprise Edition] - -* Address that clients should use to connect to the GDS Arrow Flight Server. -* defaultValue: `:8491` -* validValues: a socket address in the format `hostname:port`, `hostname` or `:port`. If missing port or hostname it is acquired from `gds.arrow.listen_address` -* dynamic: `false` - +.gds.arrow.advertised_listen_address label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Address that clients should use to connect to the GDS Arrow Flight Server. +| Default Value | `:8491` +| Valid Values | A socket address in the format `hostname:port`, `hostname` or `:port`. If missing port or hostname it is acquired from `gds.arrow.listen_address`. +| Dynamic | `false` +|=== [[gds.arrow.batch_size]] -`gds.arrow.batch_size` label:enterprise-edition[Enterprise Edition] - -* The batch size used for arrow property export. -* defaultValue: `10000` -* validValues: an integer -* dynamic: `true` +.gds.arrow.batch_size label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | The batch size used for arrow property export. +| Default Value | `10000` +| Valid Values | An integer. +| Dynamic | `true` +|=== [[gds.arrow.enabled]] -`gds.arrow.enabled` label:enterprise-edition[Enterprise Edition] - -* Enable the GDS Arrow Flight Server. -* defaultValue: `false` -* validValues: a boolean -* dynamic: `false` +.gds.arrow.enabled label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Enable the GDS Arrow Flight Server. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== [[gds.arrow.encryption.never]] -`gds.arrow.encryption.never` label:enterprise-edition[Enterprise Edition] - -* Never activate server-side encryption for the GDS Arrow Flight Server. -* defaultValue: `false` -* validValues: a boolean -* dynamic: `false` +.gds.arrow.encryption.never label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Never activate server-side encryption for the GDS Arrow Flight Server. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== [[gds.arrow.listen_address]] -`gds.arrow.listen_address` label:enterprise-edition[Enterprise Edition] - -* The maximum time to wait for the next command before aborting the import process. -* defaultValue: `10m` -* validValues: a duration (Valid units are: `ns`, `μs`, `ms`, `s`, `m`, `h` and `d` default unit is `s`) -* dynamic: `false` +.gds.arrow.listen_address label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Address the GDS Arrow Flight Server should bind to. +| Default Value | `localhost:8491` +| Valid Values | A socket address in the format `hostname:port`, `hostname` or `:port`. +| Dynamic | `false` +|=== [[gds.cluster.tx.max.size]] -`gds.cluster.tx.max.size` label:enterprise-edition[Enterprise Edition] - -* Set the maximum transaction size for GDS write back when running in Neo4j Cluster. -* defaultValue: `100000` -* validValues: an integer, must be set greater than or equal to the value of `gds.cluster.tx.min.size` -* dynamic: `false` +.gds.cluster.tx.max.size label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +| Default Value | `100000` +| Valid Values | An integer, must be set greater than or equal to the value of `gds.cluster.tx.min.size`. +| Dynamic | `false` +|=== [[gds.cluster.tx.min.size]] -`gds.cluster.tx.min.size` label:enterprise-edition[Enterprise Edition] - -* Set the minimum transaction size for GDS write back when running in Neo4j Cluster. -* defaultValue: `10000` -* validValues: an integer -* dynamic: `false` +.gds.cluster.tx.min.size label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +| Default Value | `10000` +| Valid Values | An integer. +| Dynamic | `false` +|=== [[gds.enterprise.license_file]] -`gds.enterprise.license_file` - -* Sets the location of the file that contains the Neo4j Graph Data Science library license key. -* defaultValue: `No Value` -* validValues: an absolute path -* dynamic: `false` +.gds.enterprise.license_file +[cols="1,4", caption =] +|=== +| Description | Sets the location of the file that contains the Neo4j Graph Data Science library license key. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== [[gds.export.location]] -`gds.export.location` - -* Sets the export location for file based exports. -* defaultValue: `No Value` -* validValues: an absolute path -* dynamic: `false` +.gds.export.location +[cols="1,4", caption = ] +|=== +| Description | Sets the export location for file based exports. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== [[gds.model.store_location]] -`gds.model.store_location` label:enterprise-edition[Enterprise Edition] - -* Sets the location where persisted models are stored. -* defaultValue: `No Value` -* validValues: an absolute path -* dynamic: `false` +.gds.model.store_location label:enterprise-edition[Enterprise Edition] +[cols="1,4", caption =] +|=== +| Description | Sets the location where persisted models are stored. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== [[gds.progress_tracking_enabled]] -`gds.progress_tracking_enabled` - -* Enable progress logging tracking. -* defaultValue: `true` -* validValues: a boolean -* dynamic: `false` +.gds.progress_tracking_enabled +[cols="1,4", caption =] +|=== +| Description | Enable progress logging tracking. +| Default Value | `true` +| Valid Values | A boolean. +| Dynamic | `false` +|=== [[gds.validate_using_max_memory_estimation]] -`gds.validate_using_max_memory_estimation` - -* Use maximum memory estimation in procedure memory guard. -* defaultValue: `false` -* validValues: a boolean -* dynamic: `false` +.gds.validate_using_max_memory_estimation +[cols="1,4", caption =] +|=== +| Description | Use maximum memory estimation in procedure memory guard. +| Default Value | `false` +| Valid Values | A boolean. +| Dynamic | `false` +|=== From 7d98b7b82cdc1aae80ff784553bbf8f1b192ed28 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 24 Feb 2023 10:07:33 +0000 Subject: [PATCH 216/400] Apply changes from review Co-authored-by: Ioannis Panagiotas --- .../configuration-settings.adoc | 68 +++++++++++++++---- .../configuration-settings.adoc | 23 +++---- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc index a3e02be7c91..df513f2ff1b 100644 --- a/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc +++ b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc @@ -7,21 +7,59 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 .All settings -[cols="2,1,2", caption =] -|=== -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | The maximum time to wait for the next command before aborting the import process. -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Address that clients should use to connect to the GDS Arrow Flight Server. -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | The batch size used for arrow property export. -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Enable the GDS Arrow Flight Server. -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Never activate server-side encryption for the GDS Arrow Flight Server. -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Address the GDS Arrow Flight Server should bind to. -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Set the maximum transaction size for GDS write back when running in Neo4j Cluster. -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Set the minimum transaction size for GDS write back when running in Neo4j Cluster. -<.^| <> | | Sets the location of the file that contains the Neo4j Graph Data Science library license key -<.^| <> | | Sets the export location for file based exports. -<.^| <> ^.^| label:enterprise-edition[Enterprise Edition] | Sets the location where persisted models are stored. -<.^| <> | | Enable progress logging tracking. -<.^| <> | | Use maximum memory estimation in procedure memory guard. +[cols="2,2,1", caption =] +|=== +<.^| <> +| The maximum time to wait for the next command before aborting the import process. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Address that clients should use to connect to the GDS Arrow Flight Server. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| The batch size used for arrow property export. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Enable the GDS Arrow Flight Server. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Never activate server-side encryption for the GDS Arrow Flight Server. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Address the GDS Arrow Flight Server should bind to. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Set the maximum transaction size for GDS write back when running in Neo4j Cluster. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Set the minimum transaction size for GDS write back when running in Neo4j Cluster. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Sets the location of the file that contains the Neo4j Graph Data Science library license key. +| + +<.^| <> +| Sets the export location for file based exports. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Sets the location where persisted models are stored. +^.^| label:enterprise-edition[Enterprise Edition] + +<.^| <> +| Enable progress logging tracking. +| + +<.^| <> +| Use maximum memory estimation in procedure memory guard. +| |=== [[gds.arrow.abortion_timeout]] diff --git a/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc index e59453cb88a..d576b2a4efd 100644 --- a/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc +++ b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc @@ -6,6 +6,17 @@ This page describes the available configuration settings in GDS. Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4j-conf/#neo4j-conf[neo4j.conf] file for details on how to use configuration settings. +== GDS Enterprise Edition + +.gds.enterprise.license_file +[cols="1,4", caption =] +|=== +| Description | Sets the location of the file that contains the Neo4j Graph Data Science library license key. +| Default Value | `No Value` +| Valid Values | An absolute path. +| Dynamic | `false` +|=== + [.enterprise-edition] == GDS and Arrow @@ -87,18 +98,6 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 |=== -== GDS Enterprise Edition - -.gds.enterprise.license_file -[cols="1,4", caption =] -|=== -| Description | Sets the location of the file that contains the Neo4j Graph Data Science library license key. -| Default Value | `No Value` -| Valid Values | An absolute path. -| Dynamic | `false` -|=== - - == GDS Export .gds.export.location From 6432ec57db309e70e2ee7dce29e96ad17d799e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 24 Feb 2023 11:57:59 +0100 Subject: [PATCH 217/400] Register 5.5 compat to public project They were already in a public directory but not registered in the settings or build gradle --- build.gradle | 2 ++ settings.gradle | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/build.gradle b/build.gradle index fb98ff8da7a..703608b70c2 100644 --- a/build.gradle +++ b/build.gradle @@ -33,6 +33,7 @@ ext { project(':neo4j-kernel-adapter-5.2'), project(':neo4j-kernel-adapter-5.3'), project(':neo4j-kernel-adapter-5.4'), + project(':neo4j-kernel-adapter-5.5'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -40,6 +41,7 @@ ext { project(':storage-engine-adapter-5.2'), project(':storage-engine-adapter-5.3'), project(':storage-engine-adapter-5.4'), + project(':storage-engine-adapter-5.5'), ] ] } diff --git a/settings.gradle b/settings.gradle index 83a43b9ec08..6027407577a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -136,6 +136,9 @@ project(':neo4j-kernel-adapter-5.3').projectDir = file('compatibility/5.3/neo4j- include('neo4j-kernel-adapter-5.4') project(':neo4j-kernel-adapter-5.4').projectDir = file('compatibility/5.4/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.5') +project(':neo4j-kernel-adapter-5.5').projectDir = file('compatibility/5.5/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -220,6 +223,9 @@ project(':storage-engine-adapter-5.3').projectDir = file('compatibility/5.3/stor include('storage-engine-adapter-5.4') project(':storage-engine-adapter-5.4').projectDir = file('compatibility/5.4/storage-engine-adapter') +include('storage-engine-adapter-5.5') +project(':storage-engine-adapter-5.5').projectDir = file('compatibility/5.5/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') From 6fe908f9990e2cfd58357cf11f652652548ee702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 24 Feb 2023 11:58:51 +0100 Subject: [PATCH 218/400] Document 5.5 support since 2.3.1 --- .../ROOT/pages/installation/supported-neo4j-versions.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 8dba7e593a9..d716b166490 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,6 +10,7 @@ Time to upgrade! [opts=header] |=== | Neo4j version | Neo4j Graph Data Science +| `5.5` | `2.3.1 or later` | `5.4` | `2.3`, `2.2.7 or later` | `5.3` | `2.3`, `2.2.6 or later` | `5.2` | `2.3`, `2.2.3 or later` From 95a5c76a0b8e715c193d767cc406ddfbfbadba49 Mon Sep 17 00:00:00 2001 From: Lasse Westh-Nielsen Date: Fri, 24 Feb 2023 12:19:36 +0100 Subject: [PATCH 219/400] Bump Aura version post release of 2.3.2+13 to Aura --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 651ff77fa79..612d3ba2a1a 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.2' - gdsAuraVersion = '13' + gdsAuraVersion = '14' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From d912b13326bea207a44518ebaea3b5074c9635b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 24 Feb 2023 12:42:56 +0100 Subject: [PATCH 220/400] Move default 5.5 dependency version to public --- gradle/dependencies.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index fe93cdd994b..830426d93ff 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -5,6 +5,7 @@ ext { '5.2' : properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3' : properties.getOrDefault('neo4jVersion53', '5.3.0'), '5.4' : properties.getOrDefault('neo4jVersion53', '5.4.0'), + '5.5' : properties.getOrDefault('neo4jVersion55', '5.5.0'), ] neo4jDefault = neos.'4.4' From ec9ad1dc6d43aa5f89abd909aae2f19e86452524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 24 Feb 2023 13:57:31 +0100 Subject: [PATCH 221/400] Use consecutive ids for deterministic result Before the second component id was either 1 or 3. --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index de71cfc97c3..814468a7330 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -631,7 +631,7 @@ This time, we execute WCC on `myIndexedGraph` which will allow the algorithm to .The following will run the algorithm with sampled strategy and stream results: [source, cypher, role=noplay] ---- -CALL gds.wcc.stream('myIndexedGraph', {concurrency: 1}) +CALL gds.wcc.stream('myIndexedGraph', {concurrency: 1, consecutiveIds: true}) YIELD nodeId, componentId RETURN gds.util.asNode(nodeId).name AS name, componentId ORDER BY componentId, name @@ -644,9 +644,9 @@ ORDER BY componentId, name | "Alice" | 0 | "Bridget" | 0 | "Charles" | 0 -| "Doug" | 3 -| "Mark" | 3 -| "Michael" | 3 +| "Doug" | 1 +| "Mark" | 1 +| "Michael" | 1 |=== -- From 7998250f941991e22ee1326cfe231c5f8add15e3 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 9 Feb 2023 07:55:56 +0000 Subject: [PATCH 222/400] Add test for config settings appendix page --- .../configuration-settings.adoc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc index df513f2ff1b..be7a7b63262 100644 --- a/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc +++ b/doc/modules/ROOT/pages/operations-reference/configuration-settings.adoc @@ -5,7 +5,7 @@ This page describes the available configuration settings in GDS. Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4j-conf/#neo4j-conf[neo4j.conf] file for details on how to use configuration settings. - +[.all-settings] .All settings [cols="2,2,1", caption =] |=== @@ -63,6 +63,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 |=== [[gds.arrow.abortion_timeout]] +[.setting-details] .gds.arrow.abortion_timeout label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -73,6 +74,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 |=== [[gds.arrow.advertised_listen_address]] +[.setting-details] .gds.arrow.advertised_listen_address label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -83,6 +85,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 |=== [[gds.arrow.batch_size]] +[.setting-details] .gds.arrow.batch_size label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -94,6 +97,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.arrow.enabled]] +[.setting-details] .gds.arrow.enabled label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -105,6 +109,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.arrow.encryption.never]] +[.setting-details] .gds.arrow.encryption.never label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -116,6 +121,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.arrow.listen_address]] +[.setting-details] .gds.arrow.listen_address label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -127,6 +133,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.cluster.tx.max.size]] +[.setting-details] .gds.cluster.tx.max.size label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -138,6 +145,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.cluster.tx.min.size]] +[.setting-details] .gds.cluster.tx.min.size label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -149,6 +157,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.enterprise.license_file]] +[.setting-details] .gds.enterprise.license_file [cols="1,4", caption =] |=== @@ -160,6 +169,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.export.location]] +[.setting-details] .gds.export.location [cols="1,4", caption = ] |=== @@ -171,6 +181,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.model.store_location]] +[.setting-details] .gds.model.store_location label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -182,6 +193,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.progress_tracking_enabled]] +[.setting-details] .gds.progress_tracking_enabled [cols="1,4", caption =] |=== @@ -193,6 +205,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [[gds.validate_using_max_memory_estimation]] +[.setting-details] .gds.validate_using_max_memory_estimation [cols="1,4", caption =] |=== From 623e3d0c9cd08f6671fd26bbdac1b8eb1c3d3dcb Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 24 Feb 2023 14:19:47 +0000 Subject: [PATCH 223/400] Add test for config settings production page --- .../configuration-settings.adoc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc index d576b2a4efd..ff311af117e 100644 --- a/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc +++ b/doc/modules/ROOT/pages/production-deployment/configuration-settings.adoc @@ -8,6 +8,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 == GDS Enterprise Edition +[.setting-details] .gds.enterprise.license_file [cols="1,4", caption =] |=== @@ -21,6 +22,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [.enterprise-edition] == GDS and Arrow +[.setting-details] .gds.arrow.abortion_timeout [cols="1,4", caption =] |=== @@ -30,6 +32,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 | Dynamic | `false` |=== +[.setting-details] .gds.arrow.advertised_listen_address [cols="1,4", caption =] |=== @@ -39,6 +42,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 | Dynamic | `false` |=== +[.setting-details] .gds.arrow.batch_size [cols="1,4", caption =] |=== @@ -48,6 +52,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 | Dynamic | `true` |=== +[.setting-details] .gds.arrow.enabled [cols="1,4", caption =] |=== @@ -57,6 +62,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 | Dynamic | `false` |=== +[.setting-details] .gds.arrow.encryption.never [cols="1,4", caption =] |=== @@ -66,6 +72,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 | Dynamic | `false` |=== +[.setting-details] .gds.arrow.listen_address [cols="1,4", caption =] |=== @@ -79,6 +86,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 [.enterprise-edition] == Neo4j Cluster +[.setting-details] .gds.cluster.tx.max.size [cols="1,4", caption =] |=== @@ -88,6 +96,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 | Dynamic | `false` |=== +[.setting-details] .gds.cluster.tx.min.size [cols="1,4", caption =] |=== @@ -100,6 +109,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 == GDS Export +[.setting-details] .gds.export.location [cols="1,4", caption = ] |=== @@ -109,6 +119,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 | Dynamic | `false` |=== +[.setting-details] .gds.model.store_location label:enterprise-edition[Enterprise Edition] [cols="1,4", caption =] |=== @@ -121,6 +132,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 == Miscellaneous +[.setting-details] .gds.progress_tracking_enabled [cols="1,4", caption =] |=== @@ -130,6 +142,7 @@ Refer to The https://neo4j.com/docs/operations-manual/current/configuration/neo4 | Dynamic | `false` |=== +[.setting-details] .gds.validate_using_max_memory_estimation [cols="1,4", caption =] |=== From cff88bfd971f0b39829a8f0a58baa9c24781463f Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 27 Feb 2023 10:52:53 +0100 Subject: [PATCH 224/400] Improve memory estimation for `topK > nodeCount` Co-authored-by: Veselin Nikolov --- .../neo4j/gds/similarity/nodesim/TopKMap.java | 3 ++- .../nodesim/NodeSimilarityTest.java | 24 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopKMap.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopKMap.java index 4bfde4aa305..335d66e044e 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopKMap.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopKMap.java @@ -39,10 +39,11 @@ public class TopKMap { private final BitSet sourceNodes; public static MemoryEstimation memoryEstimation(long nodes, int topK) { + int actualTopK = Math.toIntExact(Math.min(topK, nodes)); return MemoryEstimations.builder(TopKMap.class) .add("topK lists", MemoryEstimations.builder("topK lists", TopKList.class) - .add("queues", BoundedLongPriorityQueue.memoryEstimation(topK)) + .add("queues", BoundedLongPriorityQueue.memoryEstimation(actualTopK)) .build() .times(nodes) ) diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index ae9ab9fefe1..15440c06316 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -664,7 +664,7 @@ void shouldThrowForDirectionBoth(int concurrency) { } @ParameterizedTest(name = "topK = {0}") - @ValueSource(ints = {TOP_K_DEFAULT, 100}) + @ValueSource(ints = {TOP_K_DEFAULT, 100, 1_000_005}) void shouldComputeMemrec(int topK) { GraphDimensions dimensions = ImmutableGraphDimensions.builder() .nodeCount(1_000_000) @@ -773,6 +773,28 @@ void shouldComputeMemrecWithTop(int topK) { assertEquals(expected.memoryUsage(), actual.memoryUsage()); } + @Test + void shouldComputeMemrecWithTopKGreaterThanNodeCount() { + GraphDimensions dimensions = ImmutableGraphDimensions.builder() + .nodeCount(100) + .relCountUpperBound(20_000) + .build(); + + NodeSimilarityWriteConfig config = ImmutableNodeSimilarityWriteConfig + .builder() + .similarityCutoff(0.0) + .topK(200) + .writeProperty("writeProperty") + .writeRelationshipType("writeRelationshipType") + .build(); + + MemoryTree actual = new NodeSimilarityFactory<>().memoryEstimation(config).estimate(dimensions, 1); + + assertEquals(330488, actual.memoryUsage().min); + assertEquals(492088, actual.memoryUsage().max); + + } + @ParameterizedTest(name = "topK = {0}, concurrency = {1}") @MethodSource("topKAndConcurrencies") void shouldLogMessages(int topK, int concurrency) { From eafb32c599690cc905c996fc8bc6bab2d04ac471 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 27 Feb 2023 11:11:59 +0100 Subject: [PATCH 225/400] Improve memory estimation for `topN >> nodeCount` Co-authored-by: Veselin Nikolov --- .../filterednodesim/FilteredNodeSimilarityFactory.java | 6 +++++- .../gds/similarity/nodesim/NodeSimilarityFactory.java | 6 +++++- .../java/org/neo4j/gds/similarity/nodesim/TopNList.java | 9 +++++++-- .../neo4j/gds/similarity/nodesim/NodeSimilarityTest.java | 9 +++++---- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java index c4a5441d076..9e4940c9143 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/filterednodesim/FilteredNodeSimilarityFactory.java @@ -106,7 +106,11 @@ public MemoryEstimation memoryEstimation(CONFIG config) { ); } if (config.hasTopN()) { - builder.add("topN list", TopNList.memoryEstimation(topN)); + builder.add( + "topN list", + MemoryEstimations.setup("", (dimensions, concurrency) -> + TopNList.memoryEstimation(dimensions.nodeCount(), topN)) + ); } return builder.build(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java index 2f7e5c273d0..4da718731e9 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityFactory.java @@ -101,7 +101,11 @@ public MemoryEstimation memoryEstimation(CONFIG config) { ); } if (config.hasTopN()) { - builder.add("topN list", TopNList.memoryEstimation(topN)); + builder.add( + "topN list", + MemoryEstimations.setup("", (dimensions, concurrency) -> + TopNList.memoryEstimation(dimensions.nodeCount(), topN)) + ); } return builder.build(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopNList.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopNList.java index ed3f7810d4e..f1cff556e7b 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopNList.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/TopNList.java @@ -31,9 +31,14 @@ public class TopNList { - public static MemoryEstimation memoryEstimation(int topN) { + public static MemoryEstimation memoryEstimation(long nodeCount, int topN) { + int actualTopN = topN; + if (topN > nodeCount) { //avoid overflows for nodeCount * nodeCount (when nodeCount is >2^31) + long normalizedMaximum = nodeCount * nodeCount; + actualTopN = Math.toIntExact(Math.min(normalizedMaximum, topN)); + } return MemoryEstimations.builder(TopNList.class) - .add("queue", BoundedLongLongPriorityQueue.memoryEstimation(topN)) + .add("queue", BoundedLongLongPriorityQueue.memoryEstimation(actualTopN)) .build(); } diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index 15440c06316..073fe3ad12b 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -774,7 +774,7 @@ void shouldComputeMemrecWithTop(int topK) { } @Test - void shouldComputeMemrecWithTopKGreaterThanNodeCount() { + void shouldComputeMemrecWithTopKAndTopNGreaterThanNodeCount() { GraphDimensions dimensions = ImmutableGraphDimensions.builder() .nodeCount(100) .relCountUpperBound(20_000) @@ -783,15 +783,16 @@ void shouldComputeMemrecWithTopKGreaterThanNodeCount() { NodeSimilarityWriteConfig config = ImmutableNodeSimilarityWriteConfig .builder() .similarityCutoff(0.0) - .topK(200) + .topK(Integer.MAX_VALUE) + .topN(Integer.MAX_VALUE) .writeProperty("writeProperty") .writeRelationshipType("writeRelationshipType") .build(); MemoryTree actual = new NodeSimilarityFactory<>().memoryEstimation(config).estimate(dimensions, 1); - assertEquals(330488, actual.memoryUsage().min); - assertEquals(492088, actual.memoryUsage().max); + assertEquals(570592, actual.memoryUsage().min); + assertEquals(732192, actual.memoryUsage().max); } From 31d443b0a1fd4ec5a5cc1caed84c5c802f775979 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Mon, 27 Feb 2023 10:48:05 +0000 Subject: [PATCH 226/400] Update doc test example Co-Authored-By: Ioannis Panagiotas --- .../org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java | 2 +- .../ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc | 2 +- doc/modules/ROOT/pages/algorithms/node-similarity.adoc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java index 073fe3ad12b..7bdfea09fbb 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/NodeSimilarityTest.java @@ -664,7 +664,7 @@ void shouldThrowForDirectionBoth(int concurrency) { } @ParameterizedTest(name = "topK = {0}") - @ValueSource(ints = {TOP_K_DEFAULT, 100, 1_000_005}) + @ValueSource(ints = {TOP_K_DEFAULT, 100}) void shouldComputeMemrec(int topK) { GraphDimensions dimensions = ImmutableGraphDimensions.builder() .nodeCount(1_000_000) diff --git a/doc/modules/ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc b/doc/modules/ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc index 80f1edebfc3..27c53ac4838 100644 --- a/doc/modules/ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc +++ b/doc/modules/ROOT/pages/algorithms/alpha/filtered-node-similarity.adoc @@ -342,7 +342,7 @@ YIELD nodeCount, relationshipCount, bytesMin, bytesMax, requiredMemory [opts="header",cols="1,1,1,1,1"] |=== | nodeCount | relationshipCount | bytesMin | bytesMax | requiredMemory -| 9 | 9 | 2528 | 2744 | "[2528 Bytes \... 2744 Bytes]" +| 9 | 9 | 2384 | 2600 | "[2384 Bytes \... 2600 Bytes]" |=== -- [[algorithms-filtered-node-similarity-examples-stream]] diff --git a/doc/modules/ROOT/pages/algorithms/node-similarity.adoc b/doc/modules/ROOT/pages/algorithms/node-similarity.adoc index 79082b230d6..d03291d9d3d 100644 --- a/doc/modules/ROOT/pages/algorithms/node-similarity.adoc +++ b/doc/modules/ROOT/pages/algorithms/node-similarity.adoc @@ -335,7 +335,7 @@ YIELD nodeCount, relationshipCount, bytesMin, bytesMax, requiredMemory [opts="header",cols="1,1,1,1,1"] |=== | nodeCount | relationshipCount | bytesMin | bytesMax | requiredMemory -| 9 | 9 | 2528 | 2744 | "[2528 Bytes \... 2744 Bytes]" +| 9 | 9 | 2384 | 2600 | "[2384 Bytes \... 2600 Bytes]" |=== -- From fda0139d7634c201cb47c3791f1bd446ce44973b Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 28 Feb 2023 08:47:32 +0100 Subject: [PATCH 227/400] Modularity should work for communities higher than nodeCount --- .../gds/modularity/ModularityCalculator.java | 55 ++++++++++++++----- .../ModularityCalculatorFactory.java | 2 +- .../RelationshipCountCollector.java | 9 +-- .../org/neo4j/gds/louvain/LouvainTest.java | 2 +- .../modularity/ModularityCalculatorTest.java | 29 +++++----- .../RelationshipCountCollectorTest.java | 3 - .../modularity/ModularityStatsProcTest.java | 6 +- 7 files changed, 63 insertions(+), 43 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java index 5048b1ca9b8..5daf9e58aae 100644 --- a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java +++ b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculator.java @@ -19,13 +19,13 @@ */ package org.neo4j.gds.modularity; +import com.carrotsearch.hppc.cursors.LongLongCursor; import org.apache.commons.lang3.mutable.MutableDouble; -import org.apache.commons.lang3.mutable.MutableLong; import org.neo4j.gds.Algorithm; import org.neo4j.gds.api.Graph; import org.neo4j.gds.core.concurrency.RunWithConcurrency; -import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeAtomicDoubleArray; +import org.neo4j.gds.core.utils.paged.HugeLongLongMap; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.partition.PartitionUtils; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -38,17 +38,32 @@ public class ModularityCalculator extends Algorithm { private final Graph graph; private final LongUnaryOperator communityIdProvider; + private final HugeLongLongMap communityMapper; private final int concurrency; + public static ModularityCalculator create( + Graph graph, + LongUnaryOperator seedCommunityIdProvider, + int concurrency + ) { + var communityMapper = createMapping(graph.nodeCount(), seedCommunityIdProvider); + LongUnaryOperator communityIdProvider = nodeId -> communityMapper.getOrDefault( + seedCommunityIdProvider.applyAsLong(nodeId), + -1 + ); + return new ModularityCalculator(graph, communityIdProvider, communityMapper, concurrency); + } - public ModularityCalculator( + private ModularityCalculator( Graph graph, LongUnaryOperator communityIdProvider, + HugeLongLongMap communityMapper, int concurrency ) { super(ProgressTracker.NULL_TRACKER); this.graph = graph; this.communityIdProvider = communityIdProvider; + this.communityMapper = communityMapper; this.concurrency = concurrency; } @@ -56,9 +71,9 @@ public ModularityCalculator( public ModularityResult compute() { var nodeCount = graph.nodeCount(); - var insideRelationships = HugeAtomicDoubleArray.newArray(nodeCount); - var totalCommunityRelationships = HugeAtomicDoubleArray.newArray(nodeCount); - var communityTracker = HugeAtomicBitSet.create(nodeCount); + var communityCount = communityMapper.size(); + var insideRelationships = HugeAtomicDoubleArray.newArray(communityCount); + var totalCommunityRelationships = HugeAtomicDoubleArray.newArray(communityCount); var totalRelationshipWeight = new DoubleAdder(); var tasks = PartitionUtils.rangePartition( @@ -69,7 +84,6 @@ public ModularityResult compute() { graph, insideRelationships, totalCommunityRelationships, - communityTracker, communityIdProvider, totalRelationshipWeight ), Optional.empty() @@ -80,25 +94,38 @@ public ModularityResult compute() { .tasks(tasks) .run(); - var communityCount = communityTracker.cardinality(); var communityModularities = HugeObjectArray.newArray( CommunityModularity.class, communityCount ); var totalRelWeight = totalRelationshipWeight.doubleValue(); - var resultTracker = new MutableLong(); var totalModularity = new MutableDouble(); - communityTracker.forEachSetBit(communityId -> { - var ec = insideRelationships.get(communityId); - var Kc = totalCommunityRelationships.get(communityId); + long resultIndex = 0; + for (LongLongCursor cursor : communityMapper) { + long communityId = cursor.key; + long mappedCommunityId = cursor.value; + var ec = insideRelationships.get(mappedCommunityId); + var Kc = totalCommunityRelationships.get(mappedCommunityId); var modularity = (ec - Kc * Kc * (1.0 / totalRelWeight)) / totalRelWeight; totalModularity.add(modularity); - communityModularities.set(resultTracker.getAndIncrement(), CommunityModularity.of(communityId, modularity)); - }); + communityModularities.set(resultIndex++, CommunityModularity.of(communityId, modularity)); + } return ModularityResult.of(totalModularity.doubleValue(), communityCount, communityModularities); } @Override public void release() {} + static HugeLongLongMap createMapping(long nodeCount, LongUnaryOperator seedCommunityId) { + + var seedMap = new HugeLongLongMap(nodeCount); + long seedId = 0; + for (long nodeId = 0; nodeId < nodeCount; ++nodeId) { + long communityId = seedCommunityId.applyAsLong(nodeId); + if (!seedMap.containsKey(communityId)) { + seedMap.put(communityId, seedId++); + } + } + return seedMap; + } } diff --git a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculatorFactory.java b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculatorFactory.java index 7797c171606..494c9cf3ac2 100644 --- a/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculatorFactory.java +++ b/algo/src/main/java/org/neo4j/gds/modularity/ModularityCalculatorFactory.java @@ -30,7 +30,7 @@ public ModularityCalculator build( CONFIG configuration, ProgressTracker progressTracker ) { - return new ModularityCalculator( + return ModularityCalculator.create( graph, graph.nodeProperties(configuration.communityProperty())::longValue, configuration.concurrency() diff --git a/algo/src/main/java/org/neo4j/gds/modularity/RelationshipCountCollector.java b/algo/src/main/java/org/neo4j/gds/modularity/RelationshipCountCollector.java index 016d2d078d1..2483f5ba9fe 100644 --- a/algo/src/main/java/org/neo4j/gds/modularity/RelationshipCountCollector.java +++ b/algo/src/main/java/org/neo4j/gds/modularity/RelationshipCountCollector.java @@ -20,7 +20,6 @@ package org.neo4j.gds.modularity; import org.neo4j.gds.api.Graph; -import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeAtomicDoubleArray; import org.neo4j.gds.core.utils.partition.Partition; @@ -32,7 +31,6 @@ class RelationshipCountCollector implements Runnable { private final Graph localGraph; private final HugeAtomicDoubleArray insideRelationships; private final HugeAtomicDoubleArray totalCommunityRelationships; - private final HugeAtomicBitSet communityTracker; private final LongUnaryOperator communityIdProvider; private final DoubleAdder totalRelationshipWeight; @@ -42,19 +40,15 @@ class RelationshipCountCollector implements Runnable { Graph graph, HugeAtomicDoubleArray insideRelationships, HugeAtomicDoubleArray totalCommunityRelationships, - HugeAtomicBitSet communityTracker, LongUnaryOperator communityIdProvider, DoubleAdder totalRelationshipWeight ) { this.totalRelationshipWeight = totalRelationshipWeight; - assert insideRelationships.size() == totalCommunityRelationships.size() - && insideRelationships.size() == communityTracker.size(); - + assert insideRelationships.size() == totalCommunityRelationships.size(); this.partition = partition; this.localGraph = graph.concurrentCopy(); this.insideRelationships = insideRelationships; this.totalCommunityRelationships = totalCommunityRelationships; - this.communityTracker = communityTracker; this.communityIdProvider = communityIdProvider; } @@ -66,7 +60,6 @@ public void run() { long communityId = communityIdProvider.applyAsLong(nodeId); localGraph.forEachRelationship(nodeId, 1.0, (s, t, w) -> { long tCommunityId = communityIdProvider.applyAsLong(t); - communityTracker.set(communityId); if (tCommunityId == communityId) { insideRelationships.getAndAdd(communityId, w); } diff --git a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java index cb2df34ecf6..00e04bd2630 100644 --- a/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java +++ b/algo/src/test/java/org/neo4j/gds/louvain/LouvainTest.java @@ -554,7 +554,7 @@ void shouldGiveSameResultWithCalculator() { var result = louvain.compute(); assertThat(louvain.levels()).isGreaterThan(1); LongUnaryOperator vToCommunity = v -> result.getCommunity(v); - var modularityCalculator = new ModularityCalculator(myGraph, vToCommunity, 4); + var modularityCalculator = ModularityCalculator.create(myGraph, vToCommunity, 4); double calculatedModularity = modularityCalculator.compute().totalModularity(); assertThat(result.modularities()[louvain.levels() - 1]).isCloseTo(calculatedModularity, Offset.offset(1e-5)); } diff --git a/algo/src/test/java/org/neo4j/gds/modularity/ModularityCalculatorTest.java b/algo/src/test/java/org/neo4j/gds/modularity/ModularityCalculatorTest.java index 1ab530f984b..eeefba3ecbc 100644 --- a/algo/src/test/java/org/neo4j/gds/modularity/ModularityCalculatorTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularity/ModularityCalculatorTest.java @@ -19,7 +19,8 @@ */ package org.neo4j.gds.modularity; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.extension.GdlExtension; @@ -35,12 +36,12 @@ class ModularityCalculatorTest { @GdlGraph(orientation = Orientation.UNDIRECTED) static final String GRAPH = "CREATE " + - " (a1: Node { communityId: 0 })," + - " (a2: Node { communityId: 0 })," + - " (a3: Node { communityId: 5 })," + - " (a4: Node { communityId: 0 })," + - " (a5: Node { communityId: 5 })," + - " (a6: Node { communityId: 5 })," + + " (a1: Node { communityId: 0, communityId2: 200 })," + + " (a2: Node { communityId: 0, communityId2: 200 })," + + " (a3: Node { communityId: 5, communityId2: 205 })," + + " (a4: Node { communityId: 0, communityId2: 200 })," + + " (a5: Node { communityId: 5, communityId2: 205 })," + + " (a6: Node { communityId: 5, communityId2: 205 })," + " (a1)-[:R]->(a2)," + " (a1)-[:R]->(a4)," + @@ -56,11 +57,12 @@ class ModularityCalculatorTest { @Inject private GraphStore graphStore; - @Test - void compute() { - var modularityCalculator = new ModularityCalculator( + @ParameterizedTest + @CsvSource({"communityId,0", "communityId2,200"}) + void compute(String communityId, int startingId) { + var modularityCalculator = ModularityCalculator.create( graph, - graphStore.nodeProperty("communityId").values()::longValue, + graphStore.nodeProperty(communityId).values()::longValue, 4 ); @@ -74,8 +76,9 @@ void compute() { assertThat(modularities) .containsExactlyInAnyOrder( - CommunityModularity.of(0L, community_0_score), - CommunityModularity.of(5L, community_5_score) + CommunityModularity.of(startingId, community_0_score), + CommunityModularity.of(startingId + 5L, community_5_score) ); } + } diff --git a/algo/src/test/java/org/neo4j/gds/modularity/RelationshipCountCollectorTest.java b/algo/src/test/java/org/neo4j/gds/modularity/RelationshipCountCollectorTest.java index aeda650ca7b..ca4bf49c28b 100644 --- a/algo/src/test/java/org/neo4j/gds/modularity/RelationshipCountCollectorTest.java +++ b/algo/src/test/java/org/neo4j/gds/modularity/RelationshipCountCollectorTest.java @@ -21,7 +21,6 @@ import org.junit.jupiter.api.Test; import org.neo4j.gds.Orientation; -import org.neo4j.gds.core.utils.paged.HugeAtomicBitSet; import org.neo4j.gds.core.utils.paged.HugeAtomicDoubleArray; import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.partition.Partition; @@ -63,7 +62,6 @@ class RelationshipCountCollectorTest { void collect() { var insideRelationships = HugeAtomicDoubleArray.newArray(graph.nodeCount()); var totalCommunityRelationships = HugeAtomicDoubleArray.newArray(graph.nodeCount()); - var communityTracker = HugeAtomicBitSet.create(graph.nodeCount()); var communities = HugeLongArray.of(0, 0, 5, 0, 5, 5); var totalRelationshipWeight = new DoubleAdder(); @@ -72,7 +70,6 @@ void collect() { graph, insideRelationships, totalCommunityRelationships, - communityTracker, communities::get, totalRelationshipWeight ).run(); diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/modularity/ModularityStatsProcTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/modularity/ModularityStatsProcTest.java index 618085fcf93..5fac0449ecb 100644 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/modularity/ModularityStatsProcTest.java +++ b/alpha/alpha-proc/src/test/java/org/neo4j/gds/modularity/ModularityStatsProcTest.java @@ -38,10 +38,10 @@ class ModularityStatsProcTest extends BaseProcTest { @Neo4jGraph static final String GRAPH = "CREATE " + - " (a1: Node { communityId: 0 })," + - " (a2: Node { communityId: 0 })," + + " (a1: Node { communityId: 10 })," + + " (a2: Node { communityId: 10 })," + " (a3: Node { communityId: 5 })," + - " (a4: Node { communityId: 0 })," + + " (a4: Node { communityId: 10 })," + " (a5: Node { communityId: 5 })," + " (a6: Node { communityId: 5 })," + From afd89d06430b4e0b07a74422de16b838efd1a7aa Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 28 Feb 2023 12:10:29 +0100 Subject: [PATCH 228/400] Minor doc edit --- .../algorithms/alpha/modularity/specific-configuration.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc index 231f3cc695c..45968181212 100644 --- a/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc @@ -1,2 +1,2 @@ -| communityProperty | String | n/a | no | The node property that holds the community ID as an integer for each node. Note that only non-negative community IDs are considered valid and will have their conductance computed. +| communityProperty | String | n/a | no | The node property that holds the community ID as an integer for each node. Note that only non-negative community IDs are considered valid and will have their modularity score computed. | relationshipWeightProperty | String | null | yes | Relationship Weight. From 915b1b1ebd2e1ffaceb2d843908e85cf8b4775bd Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 28 Feb 2023 14:40:18 +0100 Subject: [PATCH 229/400] Fix doc test --- doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc b/doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc index 3734c8b0254..33b2e2ce995 100644 --- a/doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc +++ b/doc/modules/ROOT/pages/algorithms/alpha/modularity.adoc @@ -185,6 +185,8 @@ For more details on the stream mode in general, see xref:common-usage/running-al ---- CALL gds.alpha.modularity.stream('myGraph', { communityProperty: 'community', relationshipWeightProperty: 'weight' }) YIELD communityId, modularity +RETURN communityId, modularity +ORDER BY communityId ASC ---- .Results From 8335251ccd2d82a3312d83c0cb865395d2e0aefc Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Tue, 28 Feb 2023 15:50:40 +0100 Subject: [PATCH 230/400] Update default Neo4j 4.4.X version to 4.4.18 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 830426d93ff..9a765ee79ce 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4' : properties.getOrDefault('neo4jVersion44', '4.4.17'), + '4.4' : properties.getOrDefault('neo4jVersion44', '4.4.18'), '5.1' : properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2' : properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3' : properties.getOrDefault('neo4jVersion53', '5.3.0'), From adb81beac17cb87c42ffec5e40e3f9e0a9876b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 28 Feb 2023 17:40:47 +0100 Subject: [PATCH 231/400] Update compatibility matrix --- .../installation/supported-neo4j-versions.adoc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index d716b166490..a70ac5056b9 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -9,12 +9,11 @@ Time to upgrade! [opts=header] |=== -| Neo4j version | Neo4j Graph Data Science -| `5.5` | `2.3.1 or later` -| `5.4` | `2.3`, `2.2.7 or later` -| `5.3` | `2.3`, `2.2.6 or later` -| `5.2` | `2.3`, `2.2.3 or later` -| `5.1`| `2.3`, `2.2.1` -| `4.4.9 or later`| `2.3`, `2.2` -| `4.3.15 or later` | `2.2` +| Neo4j version | Neo4j Graph Data Science +| `5.5` | `2.3.1 or later` +| `5.4` | `2.3`, `2.2.7 or later` footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] +| `5.3` | `2.3`, `2.2.6 or later` footnote:eol[] +| `5.2` | `2.3`, `2.2.3 or later` footnote:eol[] +| `5.1` | `2.3`, `2.2.1` footnote:eol[] +| `4.4.9 or later` | `2.3`, `2.2` footnote:eol[] |=== From 0fd7a200f9d54f69cad83234cc1b44d9e9f483fb Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Wed, 1 Mar 2023 10:16:01 +0100 Subject: [PATCH 232/400] Bump supported neo4j version to 4.4.18 in public README --- README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index 2ac5017b210..0900724a380 100644 --- a/README.adoc +++ b/README.adoc @@ -84,7 +84,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.3.0 |Neo4j 5.4.0 .6+<.^|GDS 2.3.x -|Neo4j 4.4.9 - 4.4.17 +|Neo4j 4.4.9 - 4.4.18 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 From f26e7bbc66a10219488e60ab05fa1a82ef180d66 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 22 Feb 2023 11:35:50 +0100 Subject: [PATCH 233/400] Demnstrate bug --- .../gds/paths/delta/DeltaSteppingTest.java | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java index 0b7b3569cb0..2386a635b7b 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java @@ -19,7 +19,9 @@ */ package org.neo4j.gds.paths.delta; +import org.assertj.core.data.Offset; import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; @@ -29,6 +31,10 @@ import org.neo4j.gds.TestProgressTracker; import org.neo4j.gds.TestSupport; import org.neo4j.gds.api.Graph; +import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.beta.generator.PropertyProducer; +import org.neo4j.gds.beta.generator.RandomGraphGeneratorBuilder; +import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.concurrency.Pools; @@ -39,8 +45,10 @@ import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.paths.delta.config.ImmutableAllShortestPathsDeltaStreamConfig; +import org.neo4j.gds.paths.dijkstra.Dijkstra; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.DoubleStream; @@ -337,4 +345,60 @@ void singleSource(double delta, int concurrency, long idOffset) { } private DeltaSteppingTest() {} + + @RepeatedTest(200) + void incorrect() { + int nodeCount = 3_000; + long seed = 42L; + long start = 42; + int concurrency = 4; + var newGraph = new RandomGraphGeneratorBuilder() + .direction(Direction.DIRECTED) + .averageDegree(10) + .relationshipDistribution(RelationshipDistribution.POWER_LAW) + .relationshipPropertyProducer(PropertyProducer.randomDouble("foo", 1, 10)) + .nodeCount(nodeCount) + .seed(seed) + .build() + .generate(); + + var config = ImmutableAllShortestPathsDeltaStreamConfig.builder() + .concurrency(concurrency) + .sourceNode(start) + .trackRelationships(true) + .build(); + var deltaStepping = DeltaStepping.of( + newGraph, + config, + Pools.DEFAULT, + ProgressTracker.NULL_TRACKER + ).compute(); + + var dijkstraAlgo = Dijkstra + .singleSource(newGraph, config, Optional.empty(), ProgressTracker.NULL_TRACKER) + .compute(); + + double[] delta = new double[nodeCount]; + double[] djikstra = new double[nodeCount]; + + double deltaSum = 0; + double dijkstraSum = 0; + + for (var path : deltaStepping.pathSet()) { + delta[(int) path.targetNode()] = path.totalCost(); + } + for (var path : dijkstraAlgo.pathSet()) { + djikstra[(int) path.targetNode()] = path.totalCost(); + } + for (int i = 0; i < nodeCount; ++i) { + deltaSum += delta[i]; + dijkstraSum += djikstra[i]; + } + System.out.println(" deltaStepping:" + deltaSum + " dijkstra: " + dijkstraSum); + assertThat(deltaSum).isCloseTo(dijkstraSum, Offset.offset(1e-5)); + for (int i = 0; i < nodeCount; ++i) { + assertThat(djikstra[i]).isCloseTo(delta[i], Offset.offset(1e-5)); + } + } + } From 44a51dcf84d1cb8f3410bc472fc58d06387a0c1d Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 22 Feb 2023 16:14:41 +0100 Subject: [PATCH 234/400] Suggest a fix --- .../neo4j/gds/paths/delta/DeltaStepping.java | 9 ++++--- .../gds/paths/delta/TentativeDistances.java | 27 +++++++++++++------ .../gds/paths/delta/DeltaSteppingTest.java | 26 ++++++++++++++---- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaStepping.java b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaStepping.java index 23a38e6feac..7bf85bac8df 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaStepping.java +++ b/algo/src/main/java/org/neo4j/gds/paths/delta/DeltaStepping.java @@ -58,6 +58,8 @@ public final class DeltaStepping extends Algorithm { private static final int NO_BIN = Integer.MAX_VALUE; private static final int BIN_SIZE_THRESHOLD = 1000; + private static final int BATCH_SIZE = 64; + private final Graph graph; private final long startNode; @@ -281,8 +283,8 @@ int minNonEmptyBin() { private void relaxGlobalBin() { long offset; - while ((offset = frontierIndex.getAndAdd(64)) < frontierLength) { - long limit = Math.min(offset + 64, frontierLength); + while ((offset = frontierIndex.getAndAdd(BATCH_SIZE)) < frontierLength) { + long limit = Math.min(offset + BATCH_SIZE, frontierLength); for (long idx = offset; idx < limit; idx++) { var nodeId = frontier.get(idx); @@ -326,7 +328,8 @@ private void relaxNode(long nodeId) { break; } // CAX failed, retry - oldDist = witness; + //we need to fetch the most recent value from distances + oldDist = distances.distance(targetNodeId); } return true; diff --git a/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java b/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java index de82b067f5c..7653214629a 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java +++ b/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java @@ -168,24 +168,35 @@ public double compareAndExchange(long nodeId, double expectedDistance, double ne // locked by another thread if (currentPredecessor < 0) { - return distances.get(nodeId); + //we should signal failure + // for that we must be sure not to return the 'expectedDistance' by accident! + return expectedDistance / 2.0; } var witness = predecessors.compareAndExchange(nodeId, currentPredecessor, -predecessor - 1); // CAX failed if (witness != currentPredecessor) { - return distances.get(nodeId); + //we should signal failure + // for that we must be sure not to return the 'expectedDistance' by accident! + return expectedDistance / 2.0; } - // we have the look - distances.set(nodeId, newDistance); + // we have the lock; no-one else can write on nodeId at the moment. + // Let us do a check if it makes sense to update - // unlock - predecessors.set(nodeId, predecessor); + double oldDistance = distances.get(nodeId); + + if (oldDistance > newDistance) { + distances.set(nodeId, newDistance); + // unlock + predecessors.set(nodeId, predecessor); + // return previous distance to signal successful CAX - // return previous distance to signal successful CAX - return expectedDistance; + return expectedDistance; + } + predecessors.set(nodeId, currentPredecessor); + return expectedDistance / 2.0; //signal unsucesfull update } } } diff --git a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java index 2386a635b7b..785700d26ea 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java @@ -47,6 +47,8 @@ import org.neo4j.gds.paths.delta.config.ImmutableAllShortestPathsDeltaStreamConfig; import org.neo4j.gds.paths.dijkstra.Dijkstra; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.Set; @@ -346,7 +348,7 @@ void singleSource(double delta, int concurrency, long idOffset) { private DeltaSteppingTest() {} - @RepeatedTest(200) + @RepeatedTest(1000) void incorrect() { int nodeCount = 3_000; long seed = 42L; @@ -384,21 +386,35 @@ void incorrect() { double deltaSum = 0; double dijkstraSum = 0; + HashMap deltaPath = new HashMap<>(); for (var path : deltaStepping.pathSet()) { delta[(int) path.targetNode()] = path.totalCost(); + // System.out.println(path.targetNode() + ":" + Arrays.toString(path.nodeIds()) + " with " + path.totalCost()); + deltaPath.put(path.targetNode(), path.nodeIds()); } + + HashMap dijkstraPath = new HashMap<>(); for (var path : dijkstraAlgo.pathSet()) { djikstra[(int) path.targetNode()] = path.totalCost(); + // System.out.println(path.targetNode() + ":" + Arrays.toString(path.nodeIds()) + " with " + path.totalCost()); + dijkstraPath.put(path.targetNode(), path.nodeIds()); } for (int i = 0; i < nodeCount; ++i) { deltaSum += delta[i]; dijkstraSum += djikstra[i]; } - System.out.println(" deltaStepping:" + deltaSum + " dijkstra: " + dijkstraSum); - assertThat(deltaSum).isCloseTo(dijkstraSum, Offset.offset(1e-5)); - for (int i = 0; i < nodeCount; ++i) { - assertThat(djikstra[i]).isCloseTo(delta[i], Offset.offset(1e-5)); + // System.out.println(" deltaStepping:" + deltaSum + " dijkstra: " + dijkstraSum); + + for (var value : deltaPath.keySet()) { + var depath = Arrays.toString(deltaPath.get(value)); + int t = value.intValue(); + var djpath = Arrays.toString(dijkstraPath.get(value)); + // System.out.println(value + ": delta: " + depath + " dijkstra: " + djpath + " | " + delta[t] + " " + djikstra[t]); + assertThat(djikstra[t]).isCloseTo(delta[t], Offset.offset(1e-5)); + } + assertThat(deltaSum).isCloseTo(dijkstraSum, Offset.offset(1e-5)); + } } From 6c43166f53dcb64e94b1c0755a8ee64488b42571 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 24 Feb 2023 08:27:16 +0100 Subject: [PATCH 235/400] Refactor test --- .../gds/paths/delta/DeltaSteppingTest.java | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java index 785700d26ea..169ffa7cc43 100644 --- a/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java +++ b/algo/src/test/java/org/neo4j/gds/paths/delta/DeltaSteppingTest.java @@ -21,7 +21,6 @@ import org.assertj.core.data.Offset; import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; @@ -47,8 +46,6 @@ import org.neo4j.gds.paths.delta.config.ImmutableAllShortestPathsDeltaStreamConfig; import org.neo4j.gds.paths.dijkstra.Dijkstra; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.Set; @@ -348,8 +345,8 @@ void singleSource(double delta, int concurrency, long idOffset) { private DeltaSteppingTest() {} - @RepeatedTest(1000) - void incorrect() { + @Test + void shouldGiveSameResultsAsDijkstra() { int nodeCount = 3_000; long seed = 42L; long start = 42; @@ -386,31 +383,17 @@ void incorrect() { double deltaSum = 0; double dijkstraSum = 0; - HashMap deltaPath = new HashMap<>(); for (var path : deltaStepping.pathSet()) { delta[(int) path.targetNode()] = path.totalCost(); - // System.out.println(path.targetNode() + ":" + Arrays.toString(path.nodeIds()) + " with " + path.totalCost()); - deltaPath.put(path.targetNode(), path.nodeIds()); } - HashMap dijkstraPath = new HashMap<>(); for (var path : dijkstraAlgo.pathSet()) { djikstra[(int) path.targetNode()] = path.totalCost(); - // System.out.println(path.targetNode() + ":" + Arrays.toString(path.nodeIds()) + " with " + path.totalCost()); - dijkstraPath.put(path.targetNode(), path.nodeIds()); } for (int i = 0; i < nodeCount; ++i) { deltaSum += delta[i]; dijkstraSum += djikstra[i]; - } - // System.out.println(" deltaStepping:" + deltaSum + " dijkstra: " + dijkstraSum); - - for (var value : deltaPath.keySet()) { - var depath = Arrays.toString(deltaPath.get(value)); - int t = value.intValue(); - var djpath = Arrays.toString(dijkstraPath.get(value)); - // System.out.println(value + ": delta: " + depath + " dijkstra: " + djpath + " | " + delta[t] + " " + djikstra[t]); - assertThat(djikstra[t]).isCloseTo(delta[t], Offset.offset(1e-5)); + assertThat(djikstra[i]).isCloseTo(delta[i], Offset.offset(1e-5)); } assertThat(deltaSum).isCloseTo(dijkstraSum, Offset.offset(1e-5)); From 5f76e784f2973038aa3cee7caf02d68b809bdd4a Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 24 Feb 2023 08:27:32 +0100 Subject: [PATCH 236/400] Refactor result of unsuccesful operation --- .../org/neo4j/gds/paths/delta/TentativeDistances.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java b/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java index 7653214629a..3870a66d40c 100644 --- a/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java +++ b/algo/src/main/java/org/neo4j/gds/paths/delta/TentativeDistances.java @@ -170,7 +170,8 @@ public double compareAndExchange(long nodeId, double expectedDistance, double ne if (currentPredecessor < 0) { //we should signal failure // for that we must be sure not to return the 'expectedDistance' by accident! - return expectedDistance / 2.0; + //we hence return its negation (or -1 if ==0) + return (Double.compare(expectedDistance, 0.0) == 0) ? -1.0 : -expectedDistance; } var witness = predecessors.compareAndExchange(nodeId, currentPredecessor, -predecessor - 1); @@ -179,7 +180,7 @@ public double compareAndExchange(long nodeId, double expectedDistance, double ne if (witness != currentPredecessor) { //we should signal failure // for that we must be sure not to return the 'expectedDistance' by accident! - return expectedDistance / 2.0; + return (Double.compare(expectedDistance, 0.0) == 0) ? -1.0 : -expectedDistance; } // we have the lock; no-one else can write on nodeId at the moment. @@ -196,7 +197,9 @@ public double compareAndExchange(long nodeId, double expectedDistance, double ne return expectedDistance; } predecessors.set(nodeId, currentPredecessor); - return expectedDistance / 2.0; //signal unsucesfull update + //signal unsuccesful update + //note that this unsuccesful update will be the last attempt + return (Double.compare(expectedDistance, 0.0) == 0.0) ? -1.0 : -expectedDistance; } } } From f4fbf3487cc9587a13683c99f6c0106a15f58613 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 24 Feb 2023 08:27:45 +0100 Subject: [PATCH 237/400] Transfer solution to steiner tree --- .../main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/algo/src/main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java b/algo/src/main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java index eebbe85cd17..de059b4a2a0 100644 --- a/algo/src/main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java +++ b/algo/src/main/java/org/neo4j/gds/steiner/SteinerBasedDeltaTask.java @@ -150,6 +150,7 @@ private void relaxNode(long nodeId) { private void tryToUpdate(long sourceNodeId, long targetNodeId, double weight) { var oldDist = distances.distance(targetNodeId); var newDist = distances.distance(sourceNodeId) + weight; + while (Double.compare(newDist, oldDist) < 0) { var witness = distances.compareAndExchange(targetNodeId, oldDist, newDist, sourceNodeId); @@ -167,7 +168,7 @@ private void tryToUpdate(long sourceNodeId, long targetNodeId, double weight) { break; } // CAX failed, retry - oldDist = witness; + oldDist = distances.distance(targetNodeId); } smallestConsideredDistance = Math.min(newDist, smallestConsideredDistance); From 805687040da29408af996fa26f95b5a10b2f8491 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 3 Mar 2023 14:24:37 +0100 Subject: [PATCH 238/400] Reduce whitespace to increase scriptability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- gradle/dependencies.gradle | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 9a765ee79ce..e53ea3fcbc4 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,11 +1,11 @@ ext { neos = [ - '4.4' : properties.getOrDefault('neo4jVersion44', '4.4.18'), - '5.1' : properties.getOrDefault('neo4jVersion51', '5.1.0'), - '5.2' : properties.getOrDefault('neo4jVersion52', '5.2.0'), - '5.3' : properties.getOrDefault('neo4jVersion53', '5.3.0'), - '5.4' : properties.getOrDefault('neo4jVersion53', '5.4.0'), - '5.5' : properties.getOrDefault('neo4jVersion55', '5.5.0'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.18'), + '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), + '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), + '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), + '5.4': properties.getOrDefault('neo4jVersion53', '5.4.0'), + '5.5': properties.getOrDefault('neo4jVersion55', '5.5.0'), ] neo4jDefault = neos.'4.4' From c2a7869b96ff14283deefc1ceee2ff2f89c0a821 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 3 Mar 2023 14:26:33 +0100 Subject: [PATCH 239/400] Read from correct Gradle parameter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index e53ea3fcbc4..a76fa5d71c4 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -4,7 +4,7 @@ ext { '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), - '5.4': properties.getOrDefault('neo4jVersion53', '5.4.0'), + '5.4': properties.getOrDefault('neo4jVersion54', '5.4.0'), '5.5': properties.getOrDefault('neo4jVersion55', '5.5.0'), ] From aa455cd2f93502c62deeb520f2c6c517c8a76043 Mon Sep 17 00:00:00 2001 From: Paul Horn Date: Fri, 3 Mar 2023 16:27:56 +0100 Subject: [PATCH 240/400] Name all compat classes that must be in the same package differently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Jonatan Jäderberg --- .../gds/compat/_53/InMemoryMetaDataProviderImpl.java | 6 +++--- .../gds/compat/_53/InMemoryStorageEngineFactory.java | 8 ++++---- ...epository.java => InMemoryLogVersionRepository53.java} | 8 ++++---- .../InMemoryStorageCommandReaderFactory53.java} | 4 ++-- .../gds/compat/_54/InMemoryMetaDataProviderImpl.java | 6 +++--- .../gds/compat/_54/InMemoryStorageEngineFactory.java | 8 ++++---- .../recordstorage/InMemoryLogVersionRepository54.java} | 8 ++++---- .../InMemoryStorageCommandReaderFactory54.java} | 4 ++-- .../gds/compat/_55/InMemoryMetaDataProviderImpl.java | 6 +++--- .../gds/compat/_55/InMemoryStorageEngineFactory.java | 4 ++-- .../recordstorage/InMemoryLogVersionRepository55.java} | 8 ++++---- .../InMemoryStorageCommandReaderFactory55.java} | 4 ++-- 12 files changed, 37 insertions(+), 37 deletions(-) rename compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/{InMemoryLogVersionRepository.java => InMemoryLogVersionRepository53.java} (88%) rename compatibility/{5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java => 5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory53.java} (92%) rename compatibility/{5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java => 5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository54.java} (88%) rename compatibility/{5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java => 5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory54.java} (92%) rename compatibility/{5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java => 5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository55.java} (88%) rename compatibility/{5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java => 5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory55.java} (92%) diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java index 8a60b24501b..cc688fa63ff 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryMetaDataProviderImpl.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.compat._53; -import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository53; import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.kernel.KernelVersion; import org.neo4j.storageengine.api.ClosedTransactionMetadata; @@ -36,11 +36,11 @@ public class InMemoryMetaDataProviderImpl implements MetadataProvider { private final ExternalStoreId externalStoreId; - private final InMemoryLogVersionRepository logVersionRepository; + private final InMemoryLogVersionRepository53 logVersionRepository; private final InMemoryTransactionIdStoreImpl transactionIdStore; InMemoryMetaDataProviderImpl() { - this.logVersionRepository = new InMemoryLogVersionRepository(); + this.logVersionRepository = new InMemoryLogVersionRepository53(); this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); } diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java index 90d66499995..6d57e38525f 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -43,8 +43,8 @@ import org.neo4j.internal.batchimport.input.LenientStoreInput; import org.neo4j.internal.id.IdGeneratorFactory; import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; -import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; -import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository53; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory53; import org.neo4j.internal.recordstorage.StoreTokens; import org.neo4j.internal.schema.IndexConfigCompleter; import org.neo4j.internal.schema.SchemaRule; @@ -435,7 +435,7 @@ private static List unique( List tokens ) @Override public CommandReaderFactory commandReaderFactory() { - return InMemoryStorageCommandReaderFactory.INSTANCE; + return InMemoryStorageCommandReaderFactory53.INSTANCE; } @Override @@ -478,7 +478,7 @@ public TransactionIdStore readOnlyTransactionIdStore(LogTailMetadata logTailMeta @Override public LogVersionRepository readOnlyLogVersionRepository(LogTailMetadata logTailMetadata) { - return new InMemoryLogVersionRepository(); + return new InMemoryLogVersionRepository53(); } @Override diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository53.java similarity index 88% rename from compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository53.java index b491fbc7f32..7a711723491 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository53.java @@ -23,16 +23,16 @@ import java.util.concurrent.atomic.AtomicLong; -public class InMemoryLogVersionRepository implements LogVersionRepository { +public class InMemoryLogVersionRepository53 implements LogVersionRepository { private final AtomicLong logVersion; private final AtomicLong checkpointLogVersion; - public InMemoryLogVersionRepository() { - this(0,0); + public InMemoryLogVersionRepository53() { + this(0, 0); } - private InMemoryLogVersionRepository(long initialLogVersion, long initialCheckpointLogVersion) { + private InMemoryLogVersionRepository53(long initialLogVersion, long initialCheckpointLogVersion) { this.logVersion = new AtomicLong(); this.checkpointLogVersion = new AtomicLong(); this.logVersion.set(initialLogVersion); diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory53.java similarity index 92% rename from compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java rename to compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory53.java index 620a8aac11c..2804a31e179 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory53.java @@ -23,9 +23,9 @@ import org.neo4j.storageengine.api.CommandReader; import org.neo4j.storageengine.api.CommandReaderFactory; -public class InMemoryStorageCommandReaderFactory implements CommandReaderFactory { +public class InMemoryStorageCommandReaderFactory53 implements CommandReaderFactory { - public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory(); + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory53(); @Override public CommandReader get(KernelVersion kernelVersion) { diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java index 282d71ea4fb..7fd6339cb89 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryMetaDataProviderImpl.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.compat._54; -import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository54; import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.storageengine.api.ClosedTransactionMetadata; import org.neo4j.storageengine.api.ExternalStoreId; @@ -35,11 +35,11 @@ public class InMemoryMetaDataProviderImpl implements MetadataProvider { private final ExternalStoreId externalStoreId; - private final InMemoryLogVersionRepository logVersionRepository; + private final InMemoryLogVersionRepository54 logVersionRepository; private final InMemoryTransactionIdStoreImpl transactionIdStore; InMemoryMetaDataProviderImpl() { - this.logVersionRepository = new InMemoryLogVersionRepository(); + this.logVersionRepository = new InMemoryLogVersionRepository54(); this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); } diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java index 5bcba1cb8a7..a4986e7f275 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -43,8 +43,8 @@ import org.neo4j.internal.batchimport.input.LenientStoreInput; import org.neo4j.internal.id.IdGeneratorFactory; import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; -import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; -import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository54; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory54; import org.neo4j.internal.recordstorage.StoreTokens; import org.neo4j.internal.schema.IndexConfigCompleter; import org.neo4j.internal.schema.SchemaRule; @@ -449,7 +449,7 @@ private static List unique( List tokens ) @Override public CommandReaderFactory commandReaderFactory() { - return InMemoryStorageCommandReaderFactory.INSTANCE; + return InMemoryStorageCommandReaderFactory54.INSTANCE; } @Override @@ -492,7 +492,7 @@ public TransactionIdStore readOnlyTransactionIdStore(LogTailMetadata logTailMeta @Override public LogVersionRepository readOnlyLogVersionRepository(LogTailMetadata logTailMetadata) { - return new InMemoryLogVersionRepository(); + return new InMemoryLogVersionRepository54(); } @Override diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository54.java similarity index 88% rename from compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository54.java index b491fbc7f32..c901f56269d 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository54.java @@ -23,16 +23,16 @@ import java.util.concurrent.atomic.AtomicLong; -public class InMemoryLogVersionRepository implements LogVersionRepository { +public class InMemoryLogVersionRepository54 implements LogVersionRepository { private final AtomicLong logVersion; private final AtomicLong checkpointLogVersion; - public InMemoryLogVersionRepository() { - this(0,0); + public InMemoryLogVersionRepository54() { + this(0, 0); } - private InMemoryLogVersionRepository(long initialLogVersion, long initialCheckpointLogVersion) { + private InMemoryLogVersionRepository54(long initialLogVersion, long initialCheckpointLogVersion) { this.logVersion = new AtomicLong(); this.checkpointLogVersion = new AtomicLong(); this.logVersion.set(initialLogVersion); diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory54.java similarity index 92% rename from compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java rename to compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory54.java index 620a8aac11c..4d84f1e0298 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory54.java @@ -23,9 +23,9 @@ import org.neo4j.storageengine.api.CommandReader; import org.neo4j.storageengine.api.CommandReaderFactory; -public class InMemoryStorageCommandReaderFactory implements CommandReaderFactory { +public class InMemoryStorageCommandReaderFactory54 implements CommandReaderFactory { - public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory(); + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory54(); @Override public CommandReader get(KernelVersion kernelVersion) { diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java index ce7adaea00d..8dec0e77ad9 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryMetaDataProviderImpl.java @@ -19,7 +19,7 @@ */ package org.neo4j.gds.compat._55; -import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository; +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository55; import org.neo4j.io.pagecache.context.CursorContext; import org.neo4j.storageengine.api.ClosedTransactionMetadata; import org.neo4j.storageengine.api.ExternalStoreId; @@ -34,11 +34,11 @@ public class InMemoryMetaDataProviderImpl implements MetadataProvider { private final ExternalStoreId externalStoreId; - private final InMemoryLogVersionRepository logVersionRepository; + private final InMemoryLogVersionRepository55 logVersionRepository; private final InMemoryTransactionIdStoreImpl transactionIdStore; InMemoryMetaDataProviderImpl() { - this.logVersionRepository = new InMemoryLogVersionRepository(); + this.logVersionRepository = new InMemoryLogVersionRepository55(); this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); } diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java index c8c0218f65c..7c22c84e89a 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -43,7 +43,7 @@ import org.neo4j.internal.batchimport.input.LenientStoreInput; import org.neo4j.internal.id.IdGeneratorFactory; import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; -import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory55; import org.neo4j.internal.recordstorage.StoreTokens; import org.neo4j.internal.schema.IndexConfigCompleter; import org.neo4j.internal.schema.SchemaRule; @@ -446,7 +446,7 @@ private static List unique( List tokens ) @Override public CommandReaderFactory commandReaderFactory() { - return InMemoryStorageCommandReaderFactory.INSTANCE; + return InMemoryStorageCommandReaderFactory55.INSTANCE; } @Override diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository55.java similarity index 88% rename from compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java rename to compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository55.java index b491fbc7f32..cefa28655c4 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository55.java @@ -23,16 +23,16 @@ import java.util.concurrent.atomic.AtomicLong; -public class InMemoryLogVersionRepository implements LogVersionRepository { +public class InMemoryLogVersionRepository55 implements LogVersionRepository { private final AtomicLong logVersion; private final AtomicLong checkpointLogVersion; - public InMemoryLogVersionRepository() { - this(0,0); + public InMemoryLogVersionRepository55() { + this(0, 0); } - private InMemoryLogVersionRepository(long initialLogVersion, long initialCheckpointLogVersion) { + private InMemoryLogVersionRepository55(long initialLogVersion, long initialCheckpointLogVersion) { this.logVersion = new AtomicLong(); this.checkpointLogVersion = new AtomicLong(); this.logVersion.set(initialLogVersion); diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory55.java similarity index 92% rename from compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java rename to compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory55.java index 620a8aac11c..36530d653b5 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory55.java @@ -23,9 +23,9 @@ import org.neo4j.storageengine.api.CommandReader; import org.neo4j.storageengine.api.CommandReaderFactory; -public class InMemoryStorageCommandReaderFactory implements CommandReaderFactory { +public class InMemoryStorageCommandReaderFactory55 implements CommandReaderFactory { - public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory(); + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory55(); @Override public CommandReader get(KernelVersion kernelVersion) { From d2ecd5bb76ecb134f3a107b7469d1b2e8c55ef4b Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 2 Mar 2023 13:13:27 +0100 Subject: [PATCH 241/400] Allow excluding concurrency for listing of sequential algorithms Co-authored-by: Veselin Nikolov --- doc/modules/ROOT/pages/algorithms/dfs.adoc | 2 +- doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc | 3 ++- doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc | 1 + .../ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc | 1 + doc/modules/ROOT/pages/algorithms/yens.adoc | 1 + .../pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc | 1 + .../common-configuration-jobid-concurrency-entries.adoc | 3 +++ 7 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/dfs.adoc b/doc/modules/ROOT/pages/algorithms/dfs.adoc index f24525602ce..5e288f693ac 100644 --- a/doc/modules/ROOT/pages/algorithms/dfs.adoc +++ b/doc/modules/ROOT/pages/algorithms/dfs.adoc @@ -4,7 +4,7 @@ :entity: relationship :result: path in traversal order :algorithm: Depth First Search - +:sequential: true :directed: :undirected: diff --git a/doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc b/doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc index b60c6a3f05f..1dbd5ee0bfe 100644 --- a/doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc +++ b/doc/modules/ROOT/pages/algorithms/dijkstra-single-source.adoc @@ -6,6 +6,7 @@ :algorithm: Dijkstra :source-target: false :procedure-name: pass:q[gds.allShortestPaths.dijkstra] +:sequential: true :directed: @@ -25,7 +26,7 @@ To compute the shortest path between a source and a target node, xref:algorithms The GDS implementation is based on the http://www-m3.ma.tum.de/twiki/pub/MN0506/WebHome/dijkstra.pdf[original description] and uses a binary heap as priority queue. The implementation is also used for the xref:algorithms/astar.adoc[A*] and xref:algorithms/yens.adoc[Yen's] algorithms, as well as xref:algorithms/betweenness-centrality.adoc[weighted Betweenness Centrality]. -The algorithm implementation is executed using a single thread and altering the concurrency configuration has no effect. +The algorithm implementation is executed using a single thread. You can consider xref:algorithms/delta-single-source.adoc[Delta-Stepping] for an efficient parallel shortest path algorithm instead. [[algorithms-dijkstra-single-source-syntax]] diff --git a/doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc b/doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc index c9be2d4323b..8e5386debae 100644 --- a/doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc +++ b/doc/modules/ROOT/pages/algorithms/dijkstra-source-target.adoc @@ -6,6 +6,7 @@ :algorithm: Dijkstra :source-target: true :procedure-name: pass:q[gds.shortestPath.dijkstra] +:sequential: true :directed: diff --git a/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc index 47823c3b027..7637d32d767 100644 --- a/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/algorithms/minimum-weight-spanning-tree.adoc @@ -6,6 +6,7 @@ :entity: relationship :result: weight :algorithm: Prim +:sequential: true include::partial$/operations-reference/beta-note.adoc[] diff --git a/doc/modules/ROOT/pages/algorithms/yens.adoc b/doc/modules/ROOT/pages/algorithms/yens.adoc index c5bfac979ed..42558b7c059 100644 --- a/doc/modules/ROOT/pages/algorithms/yens.adoc +++ b/doc/modules/ROOT/pages/algorithms/yens.adoc @@ -6,6 +6,7 @@ :algorithm: Yen's :source-target: true :procedure-name: pass:q[gds.shortestPath.yens] +:sequential: true :directed: diff --git a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc index cc72b64e573..41ee86f71cd 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/k-minimum-weight-spanning-tree.adoc @@ -5,6 +5,7 @@ :entity: node :result: spanning tree :algorithm: k-Spanning Tree heuristic +:sequential: true include::partial$/operations-reference/alpha-note.adoc[] diff --git a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc index dcbdb16cab5..ddfe72c918b 100644 --- a/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc +++ b/doc/modules/ROOT/partials/algorithms/common-configuration/common-configuration-jobid-concurrency-entries.adoc @@ -1,3 +1,6 @@ +ifeval::[{sequential} != true] | xref:common-usage/running-algos.adoc#common-configuration-concurrency[concurrency] | Integer | 4 | yes | The number of concurrent threads used for running the algorithm. +endif::[] + | xref:common-usage/running-algos.adoc#common-configuration-jobid[jobId] | String | Generated internally | yes | An ID that can be provided to more easily track the algorithm's progress. | xref:common-usage/running-algos.adoc#common-configuration-logProgress[logProgress] | Boolean | true | yes | If disabled the progress percentage will not be logged. From c782263673f68e0a873b4b2cb2924f251c9399a3 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Mar 2023 09:07:47 +0100 Subject: [PATCH 242/400] Make k-means intro more explicit --- doc/modules/ROOT/pages/algorithms/kmeans.adoc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 8602b519735..9a1616969bf 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -15,13 +15,24 @@ include::partial$/operations-reference/beta-note.adoc[] K-Means clustering is an unsupervised learning algorithm that is used to solve clustering problems. It follows a simple procedure of classifying a given data set into a number of clusters, defined by the parameter `k`. -The clusters are then positioned as points and all observations or data points are associated with the nearest cluster, computed, adjusted and then the process starts over using the new adjustments until a desired result is reached. +The Neo4j GDS Library conducts clustering based on node properties, with a float array node property being passed as input via the `nodeProperty` parameter. +Nodes in the graph are then positioned as points in a `d`-dimensional space (where `d` is the size of length of the array property). + +The algorithm then begins by selecting `k` initial cluster centroids, which are `d`-dimensional arrays (see xref:algorithms-kmeans-sampling[section below] for more details). +The centroids act as representatives for a cluster. + +Then, all nodes in the graph calculate their Euclidean distance from each of the cluster centroids and are assigned to the cluster of minimum distance from them. + After these assignments, each cluster takes the mean of all nodes (as points) assigned to it to form its new representative centroid (as a `d`-dimensional array). + +The process repeats with the new centroids until results stabilize, i.e., only a few nodes change clusters per iteration or the number of maximum iterations is reached. + +Note that the K-Means implementation ignores relationships as it is only focused on node properties. For more information on this algorithm, see: * https://en.wikipedia.org/wiki/K-means_clustering -[[algorithms-kmeans-introduction-sampling]] +[[algorithms-kmeans-sampling]] == Initial Centroid Sampling The algorithm starts by picking `k` centroids by randomly sampling from the set of available nodes. From 79c479d91bf4e2c8e025e19cad93c5bd80e5153f Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Mar 2023 11:28:26 +0100 Subject: [PATCH 243/400] Revert "Make k-means intro more explicit" This reverts commit d8aa777bec64a1817100927af36bad63716ac8ea. --- doc/modules/ROOT/pages/algorithms/kmeans.adoc | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 9a1616969bf..8602b519735 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -15,24 +15,13 @@ include::partial$/operations-reference/beta-note.adoc[] K-Means clustering is an unsupervised learning algorithm that is used to solve clustering problems. It follows a simple procedure of classifying a given data set into a number of clusters, defined by the parameter `k`. -The Neo4j GDS Library conducts clustering based on node properties, with a float array node property being passed as input via the `nodeProperty` parameter. -Nodes in the graph are then positioned as points in a `d`-dimensional space (where `d` is the size of length of the array property). - -The algorithm then begins by selecting `k` initial cluster centroids, which are `d`-dimensional arrays (see xref:algorithms-kmeans-sampling[section below] for more details). -The centroids act as representatives for a cluster. - -Then, all nodes in the graph calculate their Euclidean distance from each of the cluster centroids and are assigned to the cluster of minimum distance from them. - After these assignments, each cluster takes the mean of all nodes (as points) assigned to it to form its new representative centroid (as a `d`-dimensional array). - -The process repeats with the new centroids until results stabilize, i.e., only a few nodes change clusters per iteration or the number of maximum iterations is reached. - -Note that the K-Means implementation ignores relationships as it is only focused on node properties. +The clusters are then positioned as points and all observations or data points are associated with the nearest cluster, computed, adjusted and then the process starts over using the new adjustments until a desired result is reached. For more information on this algorithm, see: * https://en.wikipedia.org/wiki/K-means_clustering -[[algorithms-kmeans-sampling]] +[[algorithms-kmeans-introduction-sampling]] == Initial Centroid Sampling The algorithm starts by picking `k` centroids by randomly sampling from the set of available nodes. From 0cf3221f490e86ad7b7ae61af102d6c99b8f81f1 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Mar 2023 09:07:47 +0100 Subject: [PATCH 244/400] Make k-means intro more explicit --- doc/modules/ROOT/pages/algorithms/kmeans.adoc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 8602b519735..9a1616969bf 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -15,13 +15,24 @@ include::partial$/operations-reference/beta-note.adoc[] K-Means clustering is an unsupervised learning algorithm that is used to solve clustering problems. It follows a simple procedure of classifying a given data set into a number of clusters, defined by the parameter `k`. -The clusters are then positioned as points and all observations or data points are associated with the nearest cluster, computed, adjusted and then the process starts over using the new adjustments until a desired result is reached. +The Neo4j GDS Library conducts clustering based on node properties, with a float array node property being passed as input via the `nodeProperty` parameter. +Nodes in the graph are then positioned as points in a `d`-dimensional space (where `d` is the size of length of the array property). + +The algorithm then begins by selecting `k` initial cluster centroids, which are `d`-dimensional arrays (see xref:algorithms-kmeans-sampling[section below] for more details). +The centroids act as representatives for a cluster. + +Then, all nodes in the graph calculate their Euclidean distance from each of the cluster centroids and are assigned to the cluster of minimum distance from them. + After these assignments, each cluster takes the mean of all nodes (as points) assigned to it to form its new representative centroid (as a `d`-dimensional array). + +The process repeats with the new centroids until results stabilize, i.e., only a few nodes change clusters per iteration or the number of maximum iterations is reached. + +Note that the K-Means implementation ignores relationships as it is only focused on node properties. For more information on this algorithm, see: * https://en.wikipedia.org/wiki/K-means_clustering -[[algorithms-kmeans-introduction-sampling]] +[[algorithms-kmeans-sampling]] == Initial Centroid Sampling The algorithm starts by picking `k` centroids by randomly sampling from the set of available nodes. From 6f804009af60bb5decdc8a7339ac229739058d71 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 6 Mar 2023 11:57:04 +0100 Subject: [PATCH 245/400] Make k-means intro more explicit minor --- doc/modules/ROOT/pages/algorithms/kmeans.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/kmeans.adoc b/doc/modules/ROOT/pages/algorithms/kmeans.adoc index 9a1616969bf..e080aa6a5ef 100644 --- a/doc/modules/ROOT/pages/algorithms/kmeans.adoc +++ b/doc/modules/ROOT/pages/algorithms/kmeans.adoc @@ -16,7 +16,7 @@ include::partial$/operations-reference/beta-note.adoc[] K-Means clustering is an unsupervised learning algorithm that is used to solve clustering problems. It follows a simple procedure of classifying a given data set into a number of clusters, defined by the parameter `k`. The Neo4j GDS Library conducts clustering based on node properties, with a float array node property being passed as input via the `nodeProperty` parameter. -Nodes in the graph are then positioned as points in a `d`-dimensional space (where `d` is the size of length of the array property). +Nodes in the graph are then positioned as points in a `d`-dimensional space (where `d` is the length of the array property). The algorithm then begins by selecting `k` initial cluster centroids, which are `d`-dimensional arrays (see xref:algorithms-kmeans-sampling[section below] for more details). The centroids act as representatives for a cluster. From 92a654ba2b65b6668fc575990bbc46e3e97de6aa Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Mon, 6 Mar 2023 08:52:50 +0000 Subject: [PATCH 246/400] Check type compatibility when parsing the filter --- .../gds/catalog/GraphMutateNodeLabelProc.java | 7 +- .../gds/catalog/GraphWriteNodeLabelProc.java | 7 +- .../neo4j/gds/catalog/NodeFilterParser.java | 48 ++++ .../catalog/GraphMutateNodeLabelProcTest.java | 194 +++++++++++---- .../catalog/GraphWriteNodeLabelProcTest.java | 222 ++++++++++++++++-- .../gds/catalog/NodeFilterParserTest.java | 93 ++++++++ 6 files changed, 496 insertions(+), 75 deletions(-) create mode 100644 proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeFilterParser.java create mode 100644 proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeFilterParserTest.java diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProc.java index 1f1f850c535..ab8cd73077a 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProc.java @@ -22,7 +22,6 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.ProcPreconditions; import org.neo4j.gds.beta.filter.NodesFilter; -import org.neo4j.gds.beta.filter.expression.ExpressionParser; import org.neo4j.gds.config.MutateLabelConfig; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.ProgressTimer; @@ -36,6 +35,7 @@ import java.util.concurrent.atomic.LongAdder; import java.util.stream.Stream; +import static org.neo4j.gds.catalog.NodeFilterParser.parseAndValidate; import static org.neo4j.procedure.Mode.READ; public class GraphMutateNodeLabelProc extends CatalogProc { @@ -52,14 +52,14 @@ public Stream mutate( var procedureConfig = MutateLabelConfig.of(configuration); var graphStore = graphStoreFromCatalog(graphName).graphStore(); - var filter = ExpressionParser.parse(procedureConfig.nodeFilter(), Map.of()); + var nodeFilter = parseAndValidate(graphStore, procedureConfig.nodeFilter()); var nodeLabelToMutate = NodeLabel.of(nodeLabel); var resultBuilder = MutateLabelResult.builder(graphName, nodeLabel).withConfig(procedureConfig.toMap()); try (ProgressTimer ignored = ProgressTimer.start(resultBuilder::withMutateMillis)) { var filteredNodes = NodesFilter.filterNodes( graphStore, - filter, + nodeFilter, procedureConfig.concurrency(), Map.of(), Pools.DEFAULT, @@ -87,5 +87,4 @@ public Stream mutate( return Stream.of(resultBuilder.build()); } - } diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java index fcfd27fcc7c..6aa25dcb9a5 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProc.java @@ -21,7 +21,6 @@ import org.neo4j.gds.ProcPreconditions; import org.neo4j.gds.beta.filter.NodesFilter; -import org.neo4j.gds.beta.filter.expression.ExpressionParser; import org.neo4j.gds.config.WriteLabelConfig; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.ProgressTimer; @@ -38,6 +37,7 @@ import java.util.Map; import java.util.stream.Stream; +import static org.neo4j.gds.catalog.NodeFilterParser.parseAndValidate; import static org.neo4j.procedure.Mode.WRITE; public class GraphWriteNodeLabelProc extends CatalogProc { @@ -56,16 +56,15 @@ public Stream write( ProcPreconditions.check(); var procedureConfig = WriteLabelConfig.of(configuration); - var graphStore = graphStoreFromCatalog(graphName).graphStore(); - var filter = ExpressionParser.parse(procedureConfig.nodeFilter(), Map.of()); + var nodeFilter = parseAndValidate(graphStore, procedureConfig.nodeFilter()); var resultBuilder = WriteLabelResult.builder(graphName, nodeLabel) .withConfig(procedureConfig.toMap()); try (ProgressTimer ignored = ProgressTimer.start(resultBuilder::withWriteMillis)) { var filteredNodes = NodesFilter.filterNodes( graphStore, - filter, + nodeFilter, procedureConfig.concurrency(), Map.of(), Pools.DEFAULT, diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeFilterParser.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeFilterParser.java new file mode 100644 index 00000000000..ade93552021 --- /dev/null +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/NodeFilterParser.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.catalog; + +import org.neo4j.gds.ElementProjection; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.beta.filter.expression.Expression; +import org.neo4j.gds.beta.filter.expression.ExpressionParser; +import org.neo4j.gds.beta.filter.expression.SemanticErrors; +import org.neo4j.gds.beta.filter.expression.ValidationContext; +import org.opencypher.v9_0.parser.javacc.ParseException; + +final class NodeFilterParser { + + private NodeFilterParser() {} + + static Expression parseAndValidate(GraphStore graphStore, String nodeFilter) throws IllegalArgumentException { + try { + var validationContext = ValidationContext.forNodes(graphStore); + var filter = ExpressionParser.parse( + nodeFilter.equals(ElementProjection.PROJECT_ALL) ? "true" : nodeFilter, + validationContext.availableProperties() + ); + filter.validate(validationContext).validate(); + return filter; + } catch (ParseException | SemanticErrors e) { + throw new IllegalArgumentException("Invalid `nodeFilter` expression.", e); + } + } + +} diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProcTest.java index c6e47657086..8c8e9a55ba2 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphMutateNodeLabelProcTest.java @@ -19,27 +19,37 @@ */ package org.neo4j.gds.catalog; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.graphdb.QueryExecutionException; +import java.util.Map; import java.util.concurrent.atomic.LongAdder; -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; import static org.assertj.core.api.InstanceOfAssertFactories.DOUBLE; import static org.assertj.core.api.InstanceOfAssertFactories.LONG; +@ExtendWith(SoftAssertionsExtension.class) class GraphMutateNodeLabelProcTest extends BaseProcTest { @Neo4jGraph(offsetIds = true) private static final String DB_CYPHER = "CREATE" + - " (a:A { p: 1.0 })" + - ", (b:A { p: 2.0 })" + - ", (c:A { p: 3.0 })" + + " (a:A {longProperty: 1, floatProperty: 15.5})" + + ", (b:A {longProperty: 2, floatProperty: 23})" + + ", (c:A {longProperty: 3, floatProperty: 18.3})" + ", (d:B)"; @Inject @@ -56,12 +66,12 @@ void setUp() throws Exception { } @Test - void mutateNodeLabelMultiLabelProjection() { + void mutateNodeLabelMultiLabelProjection(SoftAssertions assertions) { // Arrange runQuery( "CALL gds.graph.project('graph', " + "{" + - " A: { properties: 'p' }," + + " A: { properties: 'longProperty' }," + " B: { label: 'B' }" + "}, " + "'*')" @@ -69,24 +79,24 @@ void mutateNodeLabelMultiLabelProjection() { // Act runQuery( - "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n:A AND n.p > 1.0' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n:A AND n.longProperty > 1' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", result -> { - assertThat(result.hasNext()).isTrue(); + assertions.assertThat(result.hasNext()).isTrue(); var row = result.next(); - assertThat(row.get("nodeCount")) + assertions.assertThat(row.get("nodeCount")) .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") .isEqualTo(4L); - assertThat(row.get("nodeLabel")) + assertions.assertThat(row.get("nodeLabel")) .as("The specified node label should be present in the result") .isEqualTo("TestLabel"); - assertThat(row.get("nodeLabelsWritten")) + assertions.assertThat(row.get("nodeLabelsWritten")) .as("There should be two nodes having the new label in the in-memory graph") .isEqualTo(2L); - assertThat(result.hasNext()).isFalse(); + assertions.assertThat(result.hasNext()).isFalse(); return false; } ); @@ -94,9 +104,9 @@ void mutateNodeLabelMultiLabelProjection() { // Assert var counter = new LongAdder(); runQueryWithRowConsumer( - "CALL gds.graph.nodeProperty.stream('graph', 'p', ['TestLabel'])", + "CALL gds.graph.nodeProperty.stream('graph', 'longProperty', ['TestLabel'])", row -> { - assertThat(row.getNumber("nodeId")) + assertions.assertThat(row.getNumber("nodeId")) .asInstanceOf(LONG) .as("Only nodes `b` and `c` should have the `TestLabel` applied.") .isIn( @@ -104,45 +114,45 @@ void mutateNodeLabelMultiLabelProjection() { idFunction.of("c") ); - assertThat(row.getNumber("propertyValue")) - .asInstanceOf(DOUBLE) + assertions.assertThat(row.getNumber("propertyValue")) + .asInstanceOf(LONG) .as("The node property should match the applied filter") - .isGreaterThan(1d); + .isGreaterThan(1); counter.increment(); } ); - assertThat(counter.intValue()) + assertions.assertThat(counter.intValue()) .as("There should have been two steamed nodes") .isEqualTo(2); } @Test - void mutateNodeLabelSingleLabelProjection() { + void mutateNodeLabelSingleLabelProjection(SoftAssertions assertions) { // Arrange runQuery( "CALL gds.graph.project('graph', " + "{" + - " A: { properties: 'p' }" + + " A: { properties: 'longProperty' }" + "}, " + "'*')" ); // Act runQuery( - "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n:A AND n.p > 1.0' }) YIELD nodeCount, nodeLabelsWritten", + "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n:A AND n.longProperty > 1' }) YIELD nodeCount, nodeLabelsWritten", result -> { - assertThat(result.hasNext()).isTrue(); + assertions.assertThat(result.hasNext()).isTrue(); var row = result.next(); - assertThat(row.get("nodeCount")) + assertions.assertThat(row.get("nodeCount")) .as("Total number of nodes in the graph should be three, including the nodes that didn't get the new label") .isEqualTo(3L); - assertThat(row.get("nodeLabelsWritten")) + assertions.assertThat(row.get("nodeLabelsWritten")) .as("There should be two nodes having the new label in the in-memory graph") .isEqualTo(2L); - assertThat(result.hasNext()).isFalse(); + assertions.assertThat(result.hasNext()).isFalse(); return false; } ); @@ -150,9 +160,9 @@ void mutateNodeLabelSingleLabelProjection() { // Assert var counter = new LongAdder(); runQueryWithRowConsumer( - "CALL gds.graph.nodeProperty.stream('graph', 'p', ['TestLabel'])", + "CALL gds.graph.nodeProperty.stream('graph', 'longProperty', ['TestLabel'])", row -> { - assertThat(row.getNumber("nodeId")) + assertions.assertThat(row.getNumber("nodeId")) .asInstanceOf(LONG) .as("Only nodes `b` and `c` should have the `TestLabel` applied.") .isIn( @@ -160,47 +170,105 @@ void mutateNodeLabelSingleLabelProjection() { idFunction.of("c") ); - assertThat(row.getNumber("propertyValue")) - .asInstanceOf(DOUBLE) + assertions.assertThat(row.getNumber("propertyValue")) + .asInstanceOf(LONG) .as("The node property should match the applied filter") - .isGreaterThan(1d); + .isGreaterThan(1); counter.increment(); } ); - assertThat(counter.intValue()) + assertions.assertThat(counter.intValue()) .as("There should have been two steamed nodes") .isEqualTo(2); } @Test - void mutateNodeLabelStarProjection() { + void mutateNodeLabelStarProjection(SoftAssertions assertions) { // Arrange runQuery( "CALL gds.graph.project('graph', " + "'*'," + "'*'," + - "{ nodeProperties: {p: {defaultValue: 0.0}}}" + + "{nodeProperties: {longProperty: {defaultValue: 0}}}" + ")" ); // Act runQuery( - "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n.p > 2.0' }) YIELD nodeCount, nodeLabelsWritten", + "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n.longProperty <= 2' }) YIELD nodeCount, nodeLabelsWritten", + result -> { + assertions.assertThat(result.hasNext()).isTrue(); + + var row = result.next(); + assertions.assertThat(row.get("nodeCount")) + .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") + .isEqualTo(4L); + + assertions.assertThat(row.get("nodeLabelsWritten")) + .as("There should be three nodes having the new label in the in-memory graph") + .isEqualTo(3L); + + assertions.assertThat(result.hasNext()).isFalse(); + return false; + } + ); + + // Assert + var counter = new LongAdder(); + runQueryWithRowConsumer( + "CALL gds.graph.nodeProperty.stream('graph', 'longProperty', ['TestLabel'])", + row -> { + assertions.assertThat(row.getNumber("nodeId")) + .asInstanceOf(LONG) + .as("Nodes `a` and `b` and `d` should have the `TestLabel` applied.") + .isIn( + idFunction.of("a"), + idFunction.of("b"), + idFunction.of("d") + ); + + assertions.assertThat(row.getNumber("propertyValue")) + .asInstanceOf(LONG) + .as("The node property should match the applied filter") + .isLessThanOrEqualTo(2); + + counter.increment(); + } + ); + assertions.assertThat(counter.intValue()) + .as("There should have been three steamed nodes") + .isEqualTo(3); + } + + @Test + void shouldWorkWithFloatProperties(SoftAssertions assertions) { + // Arrange + runQuery("CALL gds.graph.project('graph', " + + "{" + + " A: { properties: 'floatProperty' }," + + " B: { label: 'B' }" + + "}, " + + "'*')"); + + + // Act + runQuery( + "CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: 'n.floatProperty <= 19.0' }) YIELD nodeCount, nodeLabelsWritten", result -> { - assertThat(result.hasNext()).isTrue(); + assertions.assertThat(result.hasNext()).isTrue(); var row = result.next(); - assertThat(row.get("nodeCount")) + assertions.assertThat(row.get("nodeCount")) .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") .isEqualTo(4L); - assertThat(row.get("nodeLabelsWritten")) + assertions.assertThat(row.get("nodeLabelsWritten")) .as("There should be two nodes having the new label in the in-memory graph") - .isEqualTo(1L); + .isEqualTo(2L); - assertThat(result.hasNext()).isFalse(); + assertions.assertThat(result.hasNext()).isFalse(); return false; } ); @@ -208,26 +276,56 @@ void mutateNodeLabelStarProjection() { // Assert var counter = new LongAdder(); runQueryWithRowConsumer( - "CALL gds.graph.nodeProperty.stream('graph', 'p', ['TestLabel'])", + "CALL gds.graph.nodeProperty.stream('graph', 'floatProperty', ['TestLabel'])", row -> { - assertThat(row.getNumber("nodeId")) + assertions.assertThat(row.getNumber("nodeId")) .asInstanceOf(LONG) - .as("Only node `c` should have the `TestLabel` applied.") - .isEqualTo( + .as("Nodes `a` and `c` should have the `TestLabel` applied.") + .isIn( + idFunction.of("a"), idFunction.of("c") ); - assertThat(row.getNumber("propertyValue")) + assertions.assertThat(row.getNumber("propertyValue")) .asInstanceOf(DOUBLE) .as("The node property should match the applied filter") - .isGreaterThan(2d); + .isLessThanOrEqualTo(19d); counter.increment(); } ); - assertThat(counter.intValue()) - .as("There should have been one steamed node") - .isEqualTo(1); + assertions.assertThat(counter.intValue()) + .as("There should have been two steamed nodes") + .isEqualTo(2); + } + + // Only check two scenarios, the rest are covered in NodeFilterParserTest. + @ParameterizedTest + @ValueSource( + strings = { + "n.floatProperty > 10", + "n.longProperty <= 19.6", + } + ) + void shouldFailOnIncompatiblePropertyAndValue(String nodeFilter) { + runQuery( + "CALL gds.graph.project('graph', " + + "{" + + " A: { properties: ['longProperty', 'floatProperty'] }," + + " B: { label: 'B' }" + + "}, " + + "'*')" + ); + + assertThatExceptionOfType(QueryExecutionException.class) + .isThrownBy(() -> runQuery("CALL gds.alpha.graph.nodeLabel.mutate('graph', 'TestLabel', { nodeFilter: $nodeFilter })", + Map.of("nodeFilter", nodeFilter))) + .withMessageContaining("Semantic errors while parsing expression") + .withMessageContaining("Incompatible types"); + } + @AfterEach + void tearDown() { + GraphStoreCatalog.removeAllLoadedGraphs(); } } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProcTest.java index 97c3d386af8..eb666460722 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphWriteNodeLabelProcTest.java @@ -19,26 +19,35 @@ */ package org.neo4j.gds.catalog; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.BaseProcTest; import org.neo4j.gds.core.loading.GraphStoreCatalog; import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.graphdb.QueryExecutionException; +import java.util.Map; import java.util.concurrent.atomic.LongAdder; -import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.assertj.core.api.InstanceOfAssertFactories.DOUBLE; import static org.assertj.core.api.InstanceOfAssertFactories.LONG; +@ExtendWith(SoftAssertionsExtension.class) class GraphWriteNodeLabelProcTest extends BaseProcTest { @Neo4jGraph(offsetIds = true) private static final String DB_CYPHER = "CREATE" + - " (a:A {p: 1})" + - ", (b:A {p: 2})" + - ", (c:A {p: 3})" + + " (a:A {longProperty: 1, floatProperty: 15.5})" + + ", (b:A {longProperty: 2, floatProperty: 23})" + + ", (c:A {longProperty: 3, floatProperty: 18.3})" + ", (d:B)"; @BeforeEach @@ -47,43 +56,44 @@ void setUp() throws Exception { GraphProjectProc.class, GraphWriteNodeLabelProc.class ); - } - @Test - void writeFilteredNodeLabels() { runQuery("CALL gds.graph.project('graph', " + "{" + - " A: { properties: 'p' }," + + " A: { properties: ['longProperty', 'floatProperty'] }," + " B: { label: 'B' }" + "}, " + "'*')"); + } + + @Test + void writeFilteredNodeLabelsWithLongPropertyGreaterComparison(SoftAssertions assertions) { // First make sure the label we want to write doesn't exist runQueryWithRowConsumer( "MATCH (n:TestLabel) RETURN count(n) AS nodeCount", - row -> assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) + row -> assertions.assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) ); runQuery( - "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.p > 1.0' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.longProperty > 1' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", result -> { - assertThat(result.hasNext()).isTrue(); + assertions.assertThat(result.hasNext()).isTrue(); var row = result.next(); - assertThat(row.get("nodeCount")) + assertions.assertThat(row.get("nodeCount")) .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") .isEqualTo(4L); - assertThat(row.get("nodeLabel")) + assertions.assertThat(row.get("nodeLabel")) .as("The specified node label should be present in the result") .isEqualTo("TestLabel"); - assertThat(row.get("nodeLabelsWritten")) + assertions.assertThat(row.get("nodeLabelsWritten")) .as("There should be two nodes having the new label written back to Neo4j") .isEqualTo(2L); - assertThat(result.hasNext()).isFalse(); + assertions.assertThat(result.hasNext()).isFalse(); return false; } ); @@ -91,21 +101,195 @@ void writeFilteredNodeLabels() { // Check that we actually created the labels in the database LongAdder rowCounter = new LongAdder(); runQueryWithRowConsumer( - "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.p AS p", + "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.longProperty AS longProperty", row -> { - assertThat(row.get("updatedLabels")) + assertions.assertThat(row.get("updatedLabels")) .asList() .containsExactlyInAnyOrder("A", "TestLabel"); - assertThat(row.getNumber("p")) + assertions.assertThat(row.getNumber("longProperty")) .asInstanceOf(LONG) .isGreaterThan(1L); rowCounter.increment(); } ); - assertThat(rowCounter.intValue()).isEqualTo(2); + assertions.assertThat(rowCounter.intValue()).isEqualTo(2); + + } + + @Test + void writeFilteredNodeLabelsWithLongPropertyLessThanOrEqualComparison(SoftAssertions assertions) { + // First make sure the label we want to write doesn't exist + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN count(n) AS nodeCount", + row -> assertions.assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) + ); + + runQuery( + "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.longProperty <= 1' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + result -> { + assertions.assertThat(result.hasNext()).isTrue(); + + var row = result.next(); + assertions.assertThat(row.get("nodeCount")) + .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") + .isEqualTo(4L); + + assertions.assertThat(row.get("nodeLabel")) + .as("The specified node label should be present in the result") + .isEqualTo("TestLabel"); + + assertions.assertThat(row.get("nodeLabelsWritten")) + .as("There should be one node having the new label written back to Neo4j") + .isEqualTo(1L); + + + assertions.assertThat(result.hasNext()).isFalse(); + return false; + } + ); + + // Check that we actually created the labels in the database + LongAdder rowCounter = new LongAdder(); + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.longProperty AS longProperty", + row -> { + assertions.assertThat(row.get("updatedLabels")) + .asList() + .containsExactlyInAnyOrder("A", "TestLabel"); + + assertions.assertThat(row.getNumber("longProperty")) + .asInstanceOf(LONG) + .isLessThanOrEqualTo(1L); + + rowCounter.increment(); + } + ); + assertions.assertThat(rowCounter.intValue()) + .as("The MATCH query should return two nodes.") + .isEqualTo(1); + } + + @Test + void writeFilteredNodeLabelsWithFloatPropertyLessThanOrEqualComparison(SoftAssertions assertions) { + // First make sure the label we want to write doesn't exist + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN count(n) AS nodeCount", + row -> assertions.assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) + ); + + runQuery( + "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.floatProperty <= 22.0' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + result -> { + assertions.assertThat(result.hasNext()).isTrue(); + + var row = result.next(); + assertions.assertThat(row.get("nodeCount")) + .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") + .isEqualTo(4L); + + assertions.assertThat(row.get("nodeLabel")) + .as("The specified node label should be present in the result") + .isEqualTo("TestLabel"); + + assertions.assertThat(row.get("nodeLabelsWritten")) + .as("There should be two nodes having the new label written back to Neo4j") + .isEqualTo(2L); + + + assertions.assertThat(result.hasNext()).isFalse(); + return false; + } + ); + + // Check that we actually created the labels in the database + LongAdder rowCounter = new LongAdder(); + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.floatProperty AS floatProperty", + row -> { + assertions.assertThat(row.get("updatedLabels")) + .asList() + .containsExactlyInAnyOrder("A", "TestLabel"); + + assertions.assertThat(row.getNumber("floatProperty")) + .asInstanceOf(DOUBLE) + .isLessThanOrEqualTo(23d); + + rowCounter.increment(); + } + ); + assertions.assertThat(rowCounter.intValue()) + .as("The MATCH query should return two nodes.") + .isEqualTo(2); + + } + + @Test + void writeFilteredNodeLabelsWithFloatPropertyGreaterComparison(SoftAssertions assertions) { + // First make sure the label we want to write doesn't exist + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN count(n) AS nodeCount", + row -> assertions.assertThat(row.getNumber("nodeCount")).asInstanceOf(LONG).isEqualTo(0L) + ); + + runQuery( + "CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: 'n:A AND n.floatProperty > 19.0' }) YIELD nodeCount, nodeLabel, nodeLabelsWritten", + result -> { + assertions.assertThat(result.hasNext()).isTrue(); + + var row = result.next(); + assertions.assertThat(row.get("nodeCount")) + .as("Total number of nodes in the graph should be four, including the nodes that didn't get the new label") + .isEqualTo(4L); + + assertions.assertThat(row.get("nodeLabel")) + .as("The specified node label should be present in the result") + .isEqualTo("TestLabel"); + + assertions.assertThat(row.get("nodeLabelsWritten")) + .as("There should be one node having the new label written back to Neo4j") + .isEqualTo(1L); + + + assertions.assertThat(result.hasNext()).isFalse(); + return false; + } + ); + + // Check that we actually created the labels in the database + LongAdder rowCounter = new LongAdder(); + runQueryWithRowConsumer( + "MATCH (n:TestLabel) RETURN labels(n) AS updatedLabels, n.floatProperty AS floatProperty", + row -> { + assertions.assertThat(row.get("updatedLabels")) + .asList() + .containsExactlyInAnyOrder("A", "TestLabel"); + + assertions.assertThat(row.getNumber("floatProperty").doubleValue()) + .isGreaterThan(19d); + + rowCounter.increment(); + } + ); + assertions.assertThat(rowCounter.intValue()).isEqualTo(1); + + } + // Only check two scenarios, the rest are covered in NodeFilterParserTest. + @ParameterizedTest + @ValueSource( + strings = { + "n.floatProperty > 10", + "n.longProperty <= 19.6", + } + ) + void shouldFailOnIncompatiblePropertyAndValue(String nodeFilter) { + assertThatExceptionOfType(QueryExecutionException.class) + .isThrownBy(() -> runQuery("CALL gds.alpha.graph.nodeLabel.write('graph', 'TestLabel', { nodeFilter: $nodeFilter })", + Map.of("nodeFilter", nodeFilter))) + .withMessageContaining("Semantic errors while parsing expression") + .withMessageContaining("Incompatible types"); } @AfterEach diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeFilterParserTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeFilterParserTest.java new file mode 100644 index 00000000000..e3e1b4cc03e --- /dev/null +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/NodeFilterParserTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.catalog; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.beta.filter.expression.SemanticErrors; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; +import static org.assertj.core.api.Assertions.assertThatNoException; + +@GdlExtension +class NodeFilterParserTest { + + @GdlGraph + private static final String DB_CYPHER = + "CREATE" + + " (a:A {longProperty: 1, floatProperty: 15.5})" + + ", (b:A {longProperty: 2, floatProperty: 23})" + + ", (c:A {longProperty: 3, floatProperty: 18.3})" + + ", (d:B)"; + + @Inject + GraphStore graphStore; + + @ParameterizedTest + @ValueSource( + strings = { + "n.floatProperty > 10", + "n.floatProperty >= 11", + "n.floatProperty < 12", + "n.floatProperty <= 19", + "n.floatProperty = 1121", + "n.longProperty > 11.0", + "n.longProperty >= 3.14", + "n.longProperty < 19.6", + "n.longProperty <= 19.6", + "n.longProperty = 21.42", + "n.longProperty <> 2.0", + } + ) + void shouldFailOnIncompatiblePropertyAndValue(String nodeFilter) { + assertThatIllegalArgumentException() + .isThrownBy(() -> NodeFilterParser.parseAndValidate(graphStore, nodeFilter)) + .withRootCauseInstanceOf(SemanticErrors.class) + .havingRootCause() + .withMessageContaining("Semantic errors while parsing expression") + .withMessageContaining("Incompatible types"); + } + + @ParameterizedTest + @ValueSource( + strings = { + "n.longProperty > 10", + "n.longProperty >= 11", + "n.longProperty < 12", + "n.longProperty <= 19", + "n.longProperty = 1121", + "n.floatProperty > 11.0", + "n.floatProperty >= 3.14", + "n.floatProperty < 19.6", + "n.floatProperty <= 19.6", + "n.floatProperty = 21.42", + "n.floatProperty <> 2.0", + } + ) + void shouldNotFailOnCompatiblePropertyAndValue(String nodeFilter) { + assertThatNoException() + .isThrownBy(() -> NodeFilterParser.parseAndValidate(graphStore, nodeFilter)); + } + +} From cb4f0aec09cd391bdd6228691ea9165d145c2374 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 23 Feb 2023 11:41:44 +0100 Subject: [PATCH 247/400] Take care of community & min. component issue --- .../ConsecutiveLongNodePropertyValues.java | 22 ++++++++++-- .../org/neo4j/gds/CommunityProcCompanion.java | 14 ++++++-- .../neo4j/gds/CommunityProcCompanionTest.java | 34 +++++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java b/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java index b5295f0f37d..7800b30618f 100644 --- a/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java +++ b/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java @@ -23,6 +23,8 @@ import org.neo4j.gds.core.utils.paged.HugeLongArray; import org.neo4j.gds.core.utils.paged.HugeLongLongMap; import org.neo4j.gds.mem.BitUtil; +import org.neo4j.values.storable.Value; +import org.neo4j.values.storable.Values; public class ConsecutiveLongNodePropertyValues implements LongNodePropertyValues { @@ -35,6 +37,7 @@ public ConsecutiveLongNodePropertyValues( long nodeCount ) { var nextConsecutiveId = -1L; + var nextSkippableId = -2L; // communities signaled with <0 are skipped and not written var setIdToConsecutiveId = new HugeLongLongMap(BitUtil.ceilDiv( nodeCount, @@ -47,8 +50,14 @@ public ConsecutiveLongNodePropertyValues( var setId = longNodeProperties.longValue(nodeId); var communityId = setIdToConsecutiveId.getOrDefault(setId, -1); if (communityId == -1) { - setIdToConsecutiveId.addTo(setId, ++nextConsecutiveId); - communityId = nextConsecutiveId; + //if this is null, it means this community should not be written + if (longNodeProperties.value(nodeId) != null) { + setIdToConsecutiveId.addTo(setId, ++nextConsecutiveId); + communityId = nextConsecutiveId; + } else { + setIdToConsecutiveId.addTo(setId, --nextSkippableId); //assign it a negative id + communityId = nextSkippableId; + } } communities.set(nodeId, communityId); } @@ -59,6 +68,15 @@ public long longValue(long nodeId) { return communities.get(nodeId); } + @Override + public Value value(long nodeId) { + if (communities.get(nodeId) < 0) { //community is skipped, return null + return null; + } else { + return Values.longValue(communities.get(nodeId)); + } + } + @Override public long size() { return communities.size(); diff --git a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index 0761d9597d4..2bf9c3e48f7 100644 --- a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -57,14 +57,23 @@ public static LongNodePropertyValues sizeFilterCheck( + LongNodePropertyValues result, + CONFIG config + ) { if (config instanceof CommunitySizeConfig) { var finalResult = result; result = ((CommunitySizeConfig) config) @@ -80,6 +89,7 @@ public static values[(int) id]); + + var config = ConfigWithComponentSize.of(CypherMapWrapper + .empty() + .withNumber("minComponentSize", 3L) + .withBoolean("consecutiveIds", true)); + + var result = CommunityProcCompanion.nodeProperties( + config, + "seed", + inputProperties, + () -> {throw new UnsupportedOperationException("Not implemented");} + ); + + + for (long i = 0L; i < result.size(); i++) { + int ii = (int) i; + if (returnedValues[ii] != null) { + assertThat(result.value(i).asObject()).isEqualTo(returnedValues[ii]); + assertThat(result.longValue(i)).isEqualTo(returnedValues[ii]); + } else { + assertThat(result.value(i)).isNull(); + assertThat(result.longValue(i)).isLessThan(0l); + } + + } + + } + private static final class TestNodePropertyValues implements LongNodePropertyValues { private final long size; private final LongToLongFunction transformer; From ef226898b12c450bd3e5488790c21b9721e62441 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 23 Feb 2023 12:37:50 +0100 Subject: [PATCH 248/400] Fix test result --- .../org/neo4j/gds/louvain/LouvainWriteProcTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/proc/community/src/test/java/org/neo4j/gds/louvain/LouvainWriteProcTest.java b/proc/community/src/test/java/org/neo4j/gds/louvain/LouvainWriteProcTest.java index f9dae068056..572fc560dea 100644 --- a/proc/community/src/test/java/org/neo4j/gds/louvain/LouvainWriteProcTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/louvain/LouvainWriteProcTest.java @@ -209,12 +209,12 @@ void zeroCommunitiesInEmptyGraph() { static Stream communitySizeInputs() { return Stream.of( // configuration | expectedCommunityCount | expectedCommunityIds - Arguments.of(Map.of("minCommunitySize", 1), 3L, new Long[] {11L, 13L, 14L}), - Arguments.of(Map.of("minCommunitySize", 3), 3L, new Long[] {14L, 13L}), - Arguments.of(Map.of("minCommunitySize", 1, "consecutiveIds", true), 3L, new Long[] {0L, 1L, 2L}), - Arguments.of(Map.of("minCommunitySize", 3, "consecutiveIds", true), 3L, new Long[] {1L, 2L}), - Arguments.of(Map.of("minCommunitySize", 1, "seedProperty", SEED_PROPERTY), 3L, new Long[] {1L, 2L, 42L}), - Arguments.of(Map.of("minCommunitySize", 3, "seedProperty", SEED_PROPERTY), 3L, new Long[] {2L, 42L}) + Arguments.of(Map.of("minCommunitySize", 1), 3L, new Long[]{11L, 13L, 14L}), + Arguments.of(Map.of("minCommunitySize", 3), 3L, new Long[]{14L, 13L}), + Arguments.of(Map.of("minCommunitySize", 1, "consecutiveIds", true), 3L, new Long[]{0L, 1L, 2L}), + Arguments.of(Map.of("minCommunitySize", 3, "consecutiveIds", true), 3L, new Long[]{0L, 1L}), + Arguments.of(Map.of("minCommunitySize", 1, "seedProperty", SEED_PROPERTY), 3L, new Long[]{1L, 2L, 42L}), + Arguments.of(Map.of("minCommunitySize", 3, "seedProperty", SEED_PROPERTY), 3L, new Long[]{2L, 42L}) ); } From ea5fc0c2219cbb84e8bd9fa7d81c63992a1fb220 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Fri, 3 Mar 2023 07:17:18 +0000 Subject: [PATCH 249/400] Make runQueryWithRowConsumer return the number of result rows --- .../src/main/java/org/neo4j/gds/BaseTest.java | 20 ++++++++--------- .../main/java/org/neo4j/gds/QueryRunner.java | 22 ++++++++++++++++--- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java index e569c861835..b0ac81be9a3 100644 --- a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java @@ -94,18 +94,18 @@ protected List allNodesWithLabel(String label) { return sourceNodes; } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( @Language("Cypher") String query, Consumer check ) { - QueryRunner.runQueryWithRowConsumer(db, query, check); + return QueryRunner.runQueryWithRowConsumer(db, query, check); } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( @Language("Cypher") String query, BiConsumer check ) { - QueryRunner.runQueryWithRowConsumer(db, query, emptyMap(), check); + return QueryRunner.runQueryWithRowConsumer(db, query, emptyMap(), check); } protected void runQueryWithRowConsumer( @@ -116,29 +116,29 @@ protected void runQueryWithRowConsumer( QueryRunner.runQueryWithRowConsumer(db, query, params, discardTx(check)); } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( GraphDatabaseService localDb, @Language("Cypher") String query, Map params, Consumer check ) { - QueryRunner.runQueryWithRowConsumer(localDb, query, params, discardTx(check)); + return QueryRunner.runQueryWithRowConsumer(localDb, query, params, discardTx(check)); } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( GraphDatabaseService localDb, @Language("Cypher") String query, Consumer check ) { - QueryRunner.runQueryWithRowConsumer(localDb, query, emptyMap(), discardTx(check)); + return QueryRunner.runQueryWithRowConsumer(localDb, query, emptyMap(), discardTx(check)); } - protected void runQueryWithRowConsumer( + protected long runQueryWithRowConsumer( String username, @Language("Cypher") String query, Consumer check ) { - QueryRunner.runQueryWithRowConsumer(db, username, query, emptyMap(), discardTx(check)); + return QueryRunner.runQueryWithRowConsumer(db, username, query, emptyMap(), discardTx(check)); } protected T runQuery( diff --git a/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java b/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java index 4d83a01e7d1..dfaa7c26aea 100644 --- a/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java +++ b/test-utils/src/main/java/org/neo4j/gds/QueryRunner.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds; +import org.apache.commons.lang3.mutable.MutableLong; import org.intellij.lang.annotations.Language; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.graphdb.GraphDatabaseService; @@ -45,53 +46,68 @@ public final class QueryRunner { private QueryRunner() {} - public static void runQueryWithRowConsumer( + public static long runQueryWithRowConsumer( GraphDatabaseService db, String username, @Language("Cypher") String query, Map params, BiConsumer rowConsumer ) { + var rowCounter = new MutableLong(); runInTransaction(db, tx -> { try (KernelTransaction.Revertable ignored = withUsername(tx, username, db.databaseName()); Result result = runQueryWithoutClosingTheResult(tx, query, params)) { result.accept(row -> { rowConsumer.accept(tx, row); + rowCounter.increment(); + return true; }); } }); + + return rowCounter.longValue(); } - public static void runQueryWithRowConsumer( + public static long runQueryWithRowConsumer( GraphDatabaseService db, @Language("Cypher") String query, Map params, BiConsumer rowConsumer ) { + var rowCounter = new MutableLong(); runInTransaction(db, tx -> { try (Result result = runQueryWithoutClosingTheResult(tx, query, params)) { result.accept(row -> { rowConsumer.accept(tx, row); + rowCounter.increment(); + return true; }); } }); + + return rowCounter.longValue(); } - public static void runQueryWithRowConsumer( + public static long runQueryWithRowConsumer( GraphDatabaseService db, @Language("Cypher") String query, Consumer rowConsumer ) { + var rowCounter = new MutableLong(); runInTransaction(db, tx -> { try (Result result = runQueryWithoutClosingTheResult(tx, query, emptyMap())) { result.accept(row -> { rowConsumer.accept(row); + rowCounter.increment(); + return true; }); } }); + + return rowCounter.longValue(); } public static void runQuery(GraphDatabaseService db, @Language("Cypher") String query) { From 3de45194efb594bfa35d707bcf2789ae2eee7c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 7 Mar 2023 11:01:32 +0100 Subject: [PATCH 250/400] Make CommunityProcCompanion a bit more grokkable Co-Authored-By: Ioannis Panagiotas Co-Authored-By: Veselin Nikolov --- .../org/neo4j/gds/CommunityProcCompanion.java | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index 2bf9c3e48f7..f0d2df57521 100644 --- a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -44,33 +44,25 @@ private CommunityProcCompanion() {} public static NodePropertyValues nodeProperties( CONFIG config, String resultProperty, - LongNodePropertyValues nodeProperties, + LongNodePropertyValues propertyValues, Supplier seedPropertySupplier ) { - var consecutiveIds = config.consecutiveIds(); var isIncremental = config.isIncremental(); var resultPropertyEqualsSeedProperty = isIncremental && resultProperty.equals(config.seedProperty()); - LongNodePropertyValues result; - if (resultPropertyEqualsSeedProperty && !consecutiveIds) { - result = LongIfChangedNodePropertyValues.of(seedPropertySupplier.get(), nodeProperties); + var incrementalNodePropertyValues = LongIfChangedNodePropertyValues.of(seedPropertySupplier.get(), propertyValues); + return communitySizeFilter(incrementalNodePropertyValues, config); } else if (consecutiveIds && !isIncremental) { - var temp = sizeFilterCheck(nodeProperties, config); - result = new ConsecutiveLongNodePropertyValues( - temp, - temp.size() - ); - return result; + var sizeFilteredPropertyValues = communitySizeFilter(propertyValues, config); + return new ConsecutiveLongNodePropertyValues(sizeFilteredPropertyValues, sizeFilteredPropertyValues.size()); } else { - result = nodeProperties; + return communitySizeFilter(propertyValues, config); } - return sizeFilterCheck(result, config); - } - private static LongNodePropertyValues sizeFilterCheck( + private static LongNodePropertyValues communitySizeFilter( LongNodePropertyValues result, CONFIG config ) { @@ -89,7 +81,6 @@ private static Date: Mon, 6 Mar 2023 14:34:26 +0000 Subject: [PATCH 251/400] Remove ComponentSizeConfig Co-authored-by: Ioannis Panagiotas --- .../org/neo4j/gds/wcc/WccStreamConfig.java | 9 +++++- .../org/neo4j/gds/wcc/WccWriteConfig.java | 10 +++++-- .../neo4j/gds/config/ComponentSizeConfig.java | 30 ------------------- .../org/neo4j/gds/CommunityProcCompanion.java | 7 ----- .../neo4j/gds/CommunityProcCompanionTest.java | 23 +++++--------- 5 files changed, 24 insertions(+), 55 deletions(-) delete mode 100644 core/src/main/java/org/neo4j/gds/config/ComponentSizeConfig.java diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccStreamConfig.java b/algo/src/main/java/org/neo4j/gds/wcc/WccStreamConfig.java index 340d05d437d..51fda2ba8a3 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccStreamConfig.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccStreamConfig.java @@ -21,12 +21,19 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.config.CommunitySizeConfig; import org.neo4j.gds.core.CypherMapWrapper; +import java.util.Optional; + @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface WccStreamConfig extends WccBaseConfig { +public interface WccStreamConfig extends WccBaseConfig, CommunitySizeConfig { + + @Override + @Configuration.Key("minComponentSize") + Optional minCommunitySize(); static WccStreamConfig of(CypherMapWrapper userInput) { WccStreamConfigImpl wccStreamConfig = new WccStreamConfigImpl(userInput); diff --git a/algo/src/main/java/org/neo4j/gds/wcc/WccWriteConfig.java b/algo/src/main/java/org/neo4j/gds/wcc/WccWriteConfig.java index 42e8ec9945c..b8eba23f7ee 100644 --- a/algo/src/main/java/org/neo4j/gds/wcc/WccWriteConfig.java +++ b/algo/src/main/java/org/neo4j/gds/wcc/WccWriteConfig.java @@ -21,14 +21,20 @@ import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.config.ComponentSizeConfig; +import org.neo4j.gds.config.CommunitySizeConfig; import org.neo4j.gds.config.WritePropertyConfig; import org.neo4j.gds.core.CypherMapWrapper; +import java.util.Optional; + @ValueClass @Configuration @SuppressWarnings("immutables:subtype") -public interface WccWriteConfig extends WccBaseConfig, WritePropertyConfig, ComponentSizeConfig { +public interface WccWriteConfig extends WccBaseConfig, WritePropertyConfig, CommunitySizeConfig { + + @Override + @Configuration.Key("minComponentSize") + Optional minCommunitySize(); static WccWriteConfig of(CypherMapWrapper userInput) { return new WccWriteConfigImpl(userInput); diff --git a/core/src/main/java/org/neo4j/gds/config/ComponentSizeConfig.java b/core/src/main/java/org/neo4j/gds/config/ComponentSizeConfig.java deleted file mode 100644 index 622ded466c6..00000000000 --- a/core/src/main/java/org/neo4j/gds/config/ComponentSizeConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.config; - -import org.neo4j.gds.annotation.Configuration; - -import java.util.Optional; - -public interface ComponentSizeConfig { - - @Configuration.LongRange(min = 1L) - Optional minComponentSize(); -} diff --git a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index f0d2df57521..b91b7d44b7d 100644 --- a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -24,7 +24,6 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.collections.HugeSparseLongArray; import org.neo4j.gds.config.CommunitySizeConfig; -import org.neo4j.gds.config.ComponentSizeConfig; import org.neo4j.gds.config.ConcurrencyConfig; import org.neo4j.gds.config.ConsecutiveIdsConfig; import org.neo4j.gds.config.SeedConfig; @@ -72,12 +71,6 @@ private static applySizeFilter(finalResult, size, config.concurrency())) .orElse(result); - } else if (config instanceof ComponentSizeConfig) { - var finalResult = result; - result = ((ComponentSizeConfig) config) - .minComponentSize() - .map(size -> applySizeFilter(finalResult, size, config.concurrency())) - .orElse(result); } return result; diff --git a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java index 695fa5ee9c5..730bd98d086 100644 --- a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java @@ -25,7 +25,7 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.NodeProperty; -import org.neo4j.gds.config.ComponentSizeConfig; +import org.neo4j.gds.config.CommunitySizeConfig; import org.neo4j.gds.config.ConcurrencyConfig; import org.neo4j.gds.config.ConsecutiveIdsConfig; import org.neo4j.gds.config.SeedConfig; @@ -94,10 +94,10 @@ void shouldReturnOnlyChangedProperties() { } @Test - void shouldRestrictComponentSize() { + void shouldRestrictCommunitySize() { LongNodePropertyValues inputProperties = new TestNodePropertyValues(10, id -> id < 5 ? id : 5 ); - var config = ConfigWithComponentSize.of(CypherMapWrapper.empty().withNumber("minComponentSize", 2L)); + var config = CommunityProcCompanionConfig.of(CypherMapWrapper.empty().withNumber("minCommunitySize", 2L)); var result = CommunityProcCompanion.nodeProperties( config, @@ -121,13 +121,13 @@ void shouldRestrictComponentSize() { @Test void shouldWorkWithMinComponentAndConsecutive() { long[] values = new long[]{20, 20, 200, 10, 10, 90, 50, 10, 50, 50, 50}; - Long[] returnedValues = new Long[]{null, null, null, 0l, 0L, null, 1l, 0l, 1l, 1l, 1l}; + Long[] returnedValues = new Long[]{null, null, null, 0L, 0L, null, 1L, 0L, 1L, 1L, 1L}; LongNodePropertyValues inputProperties = new TestNodePropertyValues(11, id -> values[(int) id]); - var config = ConfigWithComponentSize.of(CypherMapWrapper + var config = CommunityProcCompanionConfig.of(CypherMapWrapper .empty() - .withNumber("minComponentSize", 3L) + .withNumber("minCommunitySize", 3L) .withBoolean("consecutiveIds", true)); var result = CommunityProcCompanion.nodeProperties( @@ -145,7 +145,7 @@ void shouldWorkWithMinComponentAndConsecutive() { assertThat(result.longValue(i)).isEqualTo(returnedValues[ii]); } else { assertThat(result.value(i)).isNull(); - assertThat(result.longValue(i)).isLessThan(0l); + assertThat(result.longValue(i)).isLessThan(0L); } } @@ -176,16 +176,9 @@ public long longValue(long nodeId) { } @Configuration - interface CommunityProcCompanionConfig extends ConsecutiveIdsConfig, SeedConfig, ConcurrencyConfig { + interface CommunityProcCompanionConfig extends ConsecutiveIdsConfig, SeedConfig, ConcurrencyConfig, CommunitySizeConfig { static CommunityProcCompanionConfig of(CypherMapWrapper map) { return new CommunityProcCompanionConfigImpl(map); } } - - @Configuration - interface ConfigWithComponentSize extends ConsecutiveIdsConfig, SeedConfig, ComponentSizeConfig, ConcurrencyConfig { - static ConfigWithComponentSize of(CypherMapWrapper map) { - return new ConfigWithComponentSizeImpl(map); - } - } } From c92b1cc96c02dc04167f3e59cbea88c872f5dcba Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Mon, 6 Mar 2023 14:38:20 +0000 Subject: [PATCH 252/400] Document minComponentSize for WCC write Co-authored-by: Ioannis Panagiotas --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 2 +- .../partials/algorithms/wcc/specific-configuration-write.adoc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 doc/modules/ROOT/partials/algorithms/wcc/specific-configuration-write.adoc diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index 814468a7330..7dd79267831 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -190,7 +190,7 @@ include::partial$/algorithms/common-configuration/common-parameters.adoc[] |=== | Name | Type | Default | Optional | Description include::partial$/algorithms/common-configuration/common-write-configuration-entries.adoc[] -include::partial$/algorithms/wcc/specific-configuration.adoc[] +include::partial$/algorithms/wcc/specific-configuration-write.adoc[] |=== .Results diff --git a/doc/modules/ROOT/partials/algorithms/wcc/specific-configuration-write.adoc b/doc/modules/ROOT/partials/algorithms/wcc/specific-configuration-write.adoc new file mode 100644 index 00000000000..63501a43b43 --- /dev/null +++ b/doc/modules/ROOT/partials/algorithms/wcc/specific-configuration-write.adoc @@ -0,0 +1,2 @@ +include::partial$/algorithms/wcc/specific-configuration.adoc[] +| minComponentSize | Integer | 0 | yes | Only nodes inside communities larger or equal the given value will be written to the underlying Neo4j database. From 539c41b7d8a1c3f06531a72271f220635a186685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 1 Mar 2023 11:12:53 +0100 Subject: [PATCH 253/400] Avoid assert size with nodeCount The size only means the number of values stored --- .../nodeproperties/ConsecutiveLongNodePropertyValuesTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java b/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java index 860ed636ded..6edd4bccbf0 100644 --- a/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java @@ -45,7 +45,7 @@ public long longValue(long nodeId) { 10 ); - assertThat(consecutiveIds.size()).isEqualTo(10); + assertThat(consecutiveIds.size()).isEqualTo(nonConsecutiveIds.size()); for (int i = 0; i < 10; i++) { assertThat(consecutiveIds.longValue(i)).isEqualTo(i % 3); From bc07630c5d2eda4ea2d5426d36e78b48a27d8084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 1 Mar 2023 11:36:58 +0100 Subject: [PATCH 254/400] Rename size into valuesStored makes it more clear that its not actually the dimension --- .../org/neo4j/gds/scaling/ScaleProperties.java | 12 ++++++------ .../ModularityOptimization.java | 2 +- .../neo4j/gds/pagerank/EigenvectorComputation.java | 2 +- .../LongArrayPropertySimilarityComputer.java | 4 ++-- .../metrics/NullCheckingNodePropertyValues.java | 4 ++-- .../knn/GenerateRandomNeighborsTest.java | 2 +- .../knn/metrics/SimilarityComputerTest.java | 2 +- .../centrality/HarmonicCentralityWriteProc.java | 2 +- .../influenceMaximization/CelfNodeProperties.java | 2 +- .../org/neo4j/gds/scaling/ScalePropertiesProc.java | 2 +- .../main/java/org/neo4j/gds/scc/SccWriteProc.java | 2 +- .../gds/spanningtree/KSpanningTreeWriteSpec.java | 2 +- .../neo4j/gds/api/properties/PropertyValues.java | 2 +- .../properties/nodes/LongNodePropertyValues.java | 2 +- .../gds/core/huge/FilteredNodePropertyValues.java | 4 ++-- .../neo4j/gds/core/loading/NullPropertyMap.java | 4 ++-- .../gds/core/loading/ScanningNodesImporter.java | 2 +- .../DoubleArrayNodePropertiesBuilder.java | 2 +- .../DoubleNodePropertiesBuilder.java | 2 +- .../FloatArrayNodePropertiesBuilder.java | 2 +- .../LongArrayNodePropertiesBuilder.java | 2 +- .../nodeproperties/LongNodePropertiesBuilder.java | 2 +- .../gds/core/utils/IdentityPropertyValues.java | 2 +- .../core/utils/OriginalIdNodePropertyValues.java | 2 +- .../core/utils/paged/HugeAtomicDoubleArray.java | 2 +- .../gds/core/utils/paged/HugeAtomicLongArray.java | 2 +- .../neo4j/gds/core/utils/paged/HugeByteArray.java | 2 +- .../gds/core/utils/paged/HugeDoubleArray.java | 2 +- .../neo4j/gds/core/utils/paged/HugeIntArray.java | 2 +- .../neo4j/gds/core/utils/paged/HugeLongArray.java | 2 +- .../gds/core/utils/paged/HugeObjectArray.java | 6 +++--- .../core/utils/paged/dss/DisjointSetStruct.java | 2 +- .../neo4j/gds/core/loading/CSRGraphStoreTest.java | 2 +- .../gds/core/loading/HugeGraphLoadingTest.java | 4 ++-- .../NodePropertiesFromStoreBuilderTest.java | 8 ++++---- .../gds/core/utils/paged/HugeObjectArrayTest.java | 6 +++--- .../UpdatableDoubleArrayNodeProperty.java | 2 +- .../UpdatableDoubleNodeProperty.java | 2 +- .../UpdatableFloatArrayNodeProperty.java | 2 +- .../UpdatableLongArrayNodeProperty.java | 2 +- .../nodeproperties/UpdatableLongNodeProperty.java | 2 +- .../file/GraphPropertyStoreFromVisitorHelper.java | 10 +++++----- .../CsvToGraphStoreImporterIntegrationTest.java | 6 +++--- .../io/file/csv/GraphStoreToCsvExporterTest.java | 14 +++++++------- .../neo4j/gds/beta/node2vec/Node2VecCompanion.java | 2 +- .../gds/catalog/GraphDropGraphPropertiesProc.java | 2 +- .../gds/catalog/GraphDropNodePropertiesProc.java | 2 +- .../catalog/GraphDropGraphPropertiesProcTest.java | 4 ++-- .../GraphStreamGraphPropertiesProcTest.java | 8 ++++---- .../org/neo4j/gds/degree/DegreeCentralityProc.java | 2 +- .../ConsecutiveLongNodePropertyValues.java | 2 +- ...utatePropertyComputationResultConsumerTest.java | 2 +- .../org/neo4j/gds/WriteProcCancellationTest.java | 2 +- .../ConsecutiveLongNodePropertyValuesTest.java | 4 ++-- .../java/org/neo4j/gds/CommunityProcCompanion.java | 8 ++++---- .../java/org/neo4j/gds/kmeans/KmeansWriteSpec.java | 2 +- .../IntermediateCommunityNodeProperties.java | 2 +- .../java/org/neo4j/gds/louvain/LouvainProc.java | 2 +- .../LongIfChangedNodePropertyValues.java | 4 ++-- .../org/neo4j/gds/CommunityProcCompanionTest.java | 6 +++--- .../gds/embeddings/fastrp/FastRPCompanion.java | 2 +- .../embeddings/graphsage/GraphSageCompanion.java | 2 +- .../embeddings/hashgnn/HashGNNProcCompanion.java | 2 +- .../NodeClassificationPipelineMutateProc.java | 2 +- .../NodeClassificationPipelineWriteProc.java | 2 +- .../predict/NodeRegressionPipelineMutateProc.java | 2 +- .../NodeClassificationPipelineMutateProcTest.java | 6 +++--- .../NodeRegressionPipelineMutateProcTest.java | 2 +- .../org/neo4j/gds/pregel/proc/PregelBaseProc.java | 4 ++-- .../java/org/neo4j/gds/test/TestMutateProc.java | 2 +- .../DoubleArrayTestPropertyValues.java | 2 +- .../nodeproperties/DoubleTestPropertyValues.java | 2 +- .../FloatArrayTestPropertyValues.java | 2 +- .../LongArrayTestPropertyValues.java | 2 +- .../gds/nodeproperties/LongTestPropertyValues.java | 2 +- 75 files changed, 118 insertions(+), 118 deletions(-) diff --git a/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java b/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java index 1be1d2da185..ef8441eb968 100644 --- a/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java +++ b/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java @@ -225,8 +225,8 @@ public double doubleValue(long nodeId) { } @Override - public long size() { - return property.size(); + public long valuesStored() { + return property.valuesStored(); } }; } @@ -244,8 +244,8 @@ public double doubleValue(long nodeId) { } @Override - public long size() { - return property.size(); + public long valuesStored() { + return property.valuesStored(); } }; } @@ -263,8 +263,8 @@ public double doubleValue(long nodeId) { } @Override - public long size() { - return property.size(); + public long valuesStored() { + return property.valuesStored(); } }; } diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java index 686206e2346..b03f52506aa 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java @@ -416,7 +416,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long valuesStored() { return currentCommunities.size(); } }; diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java b/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java index 60cfced65c3..5ba0d03b60b 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java @@ -115,7 +115,7 @@ public boolean masterCompute(MasterComputeContext context) { var properties = new DoubleNodePropertyValues() { @Override - public long size() { + public long valuesStored() { return context.nodeCount(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java index 4f79cba6924..3f98e4b7209 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java @@ -55,7 +55,7 @@ static final class SortedLongArrayPropertyValues implements LongArrayNodePropert private final HugeObjectArray properties; SortedLongArrayPropertyValues(NodePropertyValues nodePropertyValues) { - this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.size()); + this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.valuesStored()); this.properties.setAll(i -> { var value = nodePropertyValues.longArrayValue(i).clone(); Arrays.parallelSort(value); @@ -64,7 +64,7 @@ static final class SortedLongArrayPropertyValues implements LongArrayNodePropert } @Override - public long size() { + public long valuesStored() { return properties.size(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java index 2142b87109e..d2c3228dbcf 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java @@ -86,8 +86,8 @@ public Value value(long nodeId) { } @Override - public long size() { - return properties.size(); + public long valuesStored() { + return properties.valuesStored(); } private void check(long nodeId, @Nullable Object value) { diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java index 1b2e4b4b638..2ed9a853b19 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java @@ -58,7 +58,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long valuesStored() { return nodeCount; } }; diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java index 7bec193e48a..e68a4d8863e 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java @@ -265,7 +265,7 @@ public long[] longArrayValue(long nodeId) { } @Override - public long size() { + public long valuesStored() { return nodeCount; } }; diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java index f572ba0c373..0b331e5aa2b 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java @@ -92,7 +92,7 @@ public ComputationResultConsumer floa assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(FloatArrayNodePropertyValues.class)) .describedAs("float properties must return the same size and elements as the underlying array") - .returns(floats.size(), FloatArrayNodePropertyValues::size) + .returns(floats.size(), FloatArrayNodePropertyValues::valuesStored) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.floatArrayValue(nodeId)) @@ -75,7 +75,7 @@ void shouldReturnNodePropertiesForDoubleArrayValues(HugeObjectArray do assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(DoubleArrayNodePropertyValues.class)) .describedAs("double properties must return the same size and elements as the underlying array") - .returns(doubles.size(), DoubleArrayNodePropertyValues::size) + .returns(doubles.size(), DoubleArrayNodePropertyValues::valuesStored) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.doubleArrayValue(nodeId)) @@ -104,7 +104,7 @@ void shouldReturnNodePropertiesForLongArrayValues(HugeObjectArray longs) assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(LongArrayNodePropertyValues.class)) .describedAs("long properties must return the same size and elements as the underlying array") - .returns(longs.size(), LongArrayNodePropertyValues::size) + .returns(longs.size(), LongArrayNodePropertyValues::valuesStored) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.longArrayValue(nodeId)) diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java index 456a0c2ff70..8330a59a652 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java @@ -36,7 +36,7 @@ public UpdatableDoubleArrayNodeProperty(long nodeCount, double[] defaultValue) { } @Override - public long size() { + public long valuesStored() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java index 2b486482a53..12d11cf6905 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java @@ -37,7 +37,7 @@ public UpdatableDoubleNodeProperty(long nodeCount, double defaultValue) { @Override - public long size() { + public long valuesStored() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java index 1ccaa4dd465..bc9afeafe17 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java @@ -36,7 +36,7 @@ public UpdatableFloatArrayNodeProperty(long nodeCount, float[] defaultValue) { } @Override - public long size() { + public long valuesStored() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java index 63b7475c967..29798e9d35f 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java @@ -36,7 +36,7 @@ public UpdatableLongArrayNodeProperty(long nodeCount, long[] defaultValue) { } @Override - public long size() { + public long valuesStored() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java index 7612386d2f6..37a4719f273 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java @@ -37,7 +37,7 @@ public UpdatableLongNodeProperty(long nodeCount, long defaultValue) { @Override - public long size() { + public long valuesStored() { return nodeCount; } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java index 0a7b781c7bf..5eaebfb151a 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java @@ -93,7 +93,7 @@ static class LongStreamBuilderGraphPropertyValues implements LongGraphPropertyVa } @Override - public long size() { + public long valuesStored() { return -1; } @@ -112,7 +112,7 @@ static class DoubleStreamBuilderGraphPropertyValues implements DoubleGraphProper } @Override - public long size() { + public long valuesStored() { return -1; } @@ -132,7 +132,7 @@ static class FloatArrayStreamBuilderGraphPropertyValues implements FloatArrayGra } @Override - public long size() { + public long valuesStored() { return -1; } @@ -151,7 +151,7 @@ static class DoubleArrayStreamBuilderGraphPropertyValues implements DoubleArrayG } @Override - public long size() { + public long valuesStored() { return -1; } @@ -170,7 +170,7 @@ static class LongArrayStreamBuilderGraphPropertyValues implements LongArrayGraph } @Override - public long size() { + public long valuesStored() { return -1; } diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index 65d429e88eb..28a258934ce 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -168,7 +168,7 @@ public Stream doubleArrayValues() { } @Override - public long size() { + public long valuesStored() { return 1337; } }); @@ -182,7 +182,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valuesStored() { return 10_000; } }); @@ -196,7 +196,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valuesStored() { return 10_000; } }); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java index e00324627dd..d4718f9ba22 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java @@ -235,13 +235,13 @@ void shouldExportGraphProperties() { var graphPropertyValues = new LongGraphPropertyValues() { @Override - public long size() { + public long valuesStored() { return 3; } @Override public LongStream longValues() { - return LongStream.range(0, size()); + return LongStream.range(0, valuesStored()); } }; @@ -338,13 +338,13 @@ void exportGraphPropertiesMultithreaded() throws IOException { var graphPropertyValues = new LongGraphPropertyValues() { @Override - public long size() { + public long valuesStored() { return 1_000_000; } @Override public LongStream longValues() { - return LongStream.range(0, size()); + return LongStream.range(0, valuesStored()); } }; @@ -386,7 +386,7 @@ public LongStream longValues() { .sorted() .toArray(); - assertArrayEquals(LongStream.range(0, graphPropertyValues.size()).toArray(), exportedValues); + assertArrayEquals(LongStream.range(0, graphPropertyValues.valuesStored()).toArray(), exportedValues); } @Test @@ -401,11 +401,11 @@ void exportSchemaAndDatabaseId() { graphStore.addGraphProperty("graphProp", new LongGraphPropertyValues() { @Override public LongStream longValues() { - return LongStream.range(0, size()); + return LongStream.range(0, valuesStored()); } @Override - public long size() { + public long valuesStored() { return 3; } }); diff --git a/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java b/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java index c155cd16a49..424eb7ac745 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java +++ b/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java @@ -40,7 +40,7 @@ static NodePropertyValues nodeProperties( return new FloatArrayNodePropertyValues() { @Override - public long size() { + public long valuesStored() { return size; } diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java index b2b99e2aa3e..61ceffbe9fa 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java @@ -57,7 +57,7 @@ public Stream run( config.validate(graphStore); // removing - long propertiesRemoved = graphStore.graphPropertyValues(graphProperty).size(); + long propertiesRemoved = graphStore.graphPropertyValues(graphProperty).valuesStored(); runWithExceptionLogging( "Graph property removal failed", () -> graphStore.removeGraphProperty(graphProperty) diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java index cfbef0cb63a..9c2a54e5804 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java @@ -120,7 +120,7 @@ private Long dropNodeProperties( progressTracker.beginSubTask(); config.nodeProperties().forEach(propertyKey -> { - removedPropertiesCount.add(graphStore.nodeProperty(propertyKey).values().size()); + removedPropertiesCount.add(graphStore.nodeProperty(propertyKey).values().valuesStored()); graphStore.removeNodeProperty(propertyKey); progressTracker.logProgress(); }); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java index b9b34fd52bc..d3ac5037215 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java @@ -68,7 +68,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valuesStored() { return 10; } @@ -113,7 +113,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valuesStored() { return 10; } diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java index b417244e63b..1cc2e1b147b 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java @@ -92,7 +92,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valuesStored() { return 10; } @@ -103,7 +103,7 @@ public DoubleStream doubleValues() { } @Override - public long size() { + public long valuesStored() { return 3; } }), arguments(new LongArrayGraphPropertyValues() { @@ -113,7 +113,7 @@ public Stream longArrayValues() { } @Override - public long size() { + public long valuesStored() { return 2; } })); @@ -137,7 +137,7 @@ public LongStream longValues() { } @Override - public long size() { + public long valuesStored() { return 10; } diff --git a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java index 5620d7b0368..bb389cb5228 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java @@ -52,7 +52,7 @@ static NodePropertyValues nodeProperties(ComputationResult getCom class TestNodePropertyValues implements LongNodePropertyValues { @Override - public long size() { + public long valuesStored() { return graphStore.nodeCount(); } diff --git a/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java b/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java index 3252bcdfca9..b7447feac17 100644 --- a/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java @@ -59,7 +59,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long valuesStored() { return 42; } }; diff --git a/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java b/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java index 6edd4bccbf0..b2bafba8649 100644 --- a/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java @@ -30,7 +30,7 @@ class ConsecutiveLongNodePropertyValuesTest { void shouldReturnConsecutiveIds() { LongNodePropertyValues nonConsecutiveIds = new LongNodePropertyValues() { @Override - public long size() { + public long valuesStored() { return 10; } @@ -45,7 +45,7 @@ public long longValue(long nodeId) { 10 ); - assertThat(consecutiveIds.size()).isEqualTo(nonConsecutiveIds.size()); + assertThat(consecutiveIds.valuesStored()).isEqualTo(nonConsecutiveIds.valuesStored()); for (int i = 0; i < 10; i++) { assertThat(consecutiveIds.longValue(i)).isEqualTo(i % 3); diff --git a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index b91b7d44b7d..04548ab8e89 100644 --- a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -55,7 +55,7 @@ public static NodePropertyValues nodeProperties( return new LongArrayNodePropertyValues() { @Override - public long size() { + public long valuesStored() { return size; } diff --git a/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java b/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java index 538c704f510..390a310a5ab 100644 --- a/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java +++ b/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java @@ -76,7 +76,7 @@ public Value value(long nodeId) { } @Override - public long size() { - return newProperties.size(); + public long valuesStored() { + return newProperties.valuesStored(); } } diff --git a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java index 730bd98d086..413b5c79b73 100644 --- a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java @@ -87,7 +87,7 @@ void shouldReturnOnlyChangedProperties() { ); assertThat(result).isInstanceOf(LongIfChangedNodePropertyValues.class); - for (long i = 0; i < result.size(); i++) { + for (long i = 0; i < result.valuesStored(); i++) { assertThat(result.longValue(i)).isEqualTo(inputProperties.longValue(i)); assertThat(result.value(i)).isNull(); } @@ -106,7 +106,7 @@ void shouldRestrictCommunitySize() { () -> { throw new UnsupportedOperationException("Not implemented"); } ); - for (long i = 0L; i < result.size(); i++) { + for (long i = 0L; i < result.valuesStored(); i++) { if (i < 5) { assertThat(result.longValue(i)).isEqualTo(inputProperties.longValue(i)); @@ -165,7 +165,7 @@ private TestNodePropertyValues( } @Override - public long size() { + public long valuesStored() { return size; } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java index b5de01300bb..541df01a6a2 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java @@ -40,7 +40,7 @@ public float[] floatArrayValue(long nodeId) { } @Override - public long size() { + public long valuesStored() { return nodeCount; } }; diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java index 99026545670..fb85c6beef0 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java @@ -46,7 +46,7 @@ public static DoubleArrayNodePropertyValues getN return new DoubleArrayNodePropertyValues() { @Override - public long size() { + public long valuesStored() { return size; } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java index e8ed5edbb79..719f5ae205a 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java @@ -40,7 +40,7 @@ public double[] doubleArrayValue(long nodeId) { } @Override - public long size() { + public long valuesStored() { return nodeCount; } }; diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java index b0d05fc5451..4149076c1ac 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java @@ -97,7 +97,7 @@ protected List nodePropertyList(ComputationResult { var properties = new DoubleArrayNodePropertyValues() { @Override - public long size() { + public long valuesStored() { return computationResult.graph().nodeCount(); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java index 283f6e9bc2c..474cbaf54bc 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java @@ -98,7 +98,7 @@ protected List nodePropertyList(ComputationResult { var properties = new DoubleArrayNodePropertyValues() { @Override - public long size() { + public long valuesStored() { return computationResult.graph().nodeCount(); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java index ac79e6e6aeb..5de1105d9ca 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java @@ -71,7 +71,7 @@ protected NodePropertyValues nodeProperties(ComputationResult longArrays) {this.longArrays = longArrays;} @Override - public long size() { + public long valuesStored() { return longArrays.size(); } @@ -180,7 +180,7 @@ static class HugeObjectArrayDoubleArrayPropertyValues implements DoubleArrayNode HugeObjectArrayDoubleArrayPropertyValues(HugeObjectArray doubleArrays) {this.doubleArrays = doubleArrays;} @Override - public long size() { + public long valuesStored() { return doubleArrays.size(); } diff --git a/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java b/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java index 18d88554f2b..1ba40e5ffe4 100644 --- a/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java +++ b/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java @@ -70,7 +70,7 @@ public long longValue(long nodeId) { } @Override - public long size() { + public long valuesStored() { return 0; } }; diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java index 0af1f739c3c..d04d4546946 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java @@ -28,7 +28,7 @@ public final class DoubleArrayTestPropertyValues implements DoubleArrayNodePrope public DoubleArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long size() { + public long valuesStored() { return 0; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java index 5088f7f5b4a..5d97154a815 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java @@ -29,7 +29,7 @@ public final class DoubleTestPropertyValues implements DoubleNodePropertyValues public DoubleTestPropertyValues(LongToDoubleFunction transformer) {this.transformer = transformer;} @Override - public long size() { + public long valuesStored() { return 0; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java index f2f8e723075..97a90fff1af 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java @@ -28,7 +28,7 @@ public final class FloatArrayTestPropertyValues implements FloatArrayNodePropert public FloatArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long size() { + public long valuesStored() { return 0; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java index 57bf4700de3..631e0d4ee10 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java @@ -28,7 +28,7 @@ public final class LongArrayTestPropertyValues implements LongArrayNodePropertyV public LongArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long size() { + public long valuesStored() { return 0; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java index 9ad961cece0..3dc14a8663d 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java @@ -28,7 +28,7 @@ public final class LongTestPropertyValues implements LongNodePropertyValues { public LongTestPropertyValues(LongToLongFunction transformer) {this.transformer = transformer;} @Override - public long size() { + public long valuesStored() { return 0; } From 9ac61e3fec87f472f43bcbb3b1ffca232beba163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 2 Mar 2023 15:37:03 +0100 Subject: [PATCH 255/400] Prove bug in current size() usage --- .../LongArrayPropertySimilarityComputer.java | 8 ++- ...ngArrayPropertySimilarityComputerTest.java | 52 +++++++++++++++++++ .../gds/similarity/knn/KnnStreamProcTest.java | 28 ++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java index 3f98e4b7209..61c59cb330b 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java @@ -57,7 +57,13 @@ static final class SortedLongArrayPropertyValues implements LongArrayNodePropert SortedLongArrayPropertyValues(NodePropertyValues nodePropertyValues) { this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.valuesStored()); this.properties.setAll(i -> { - var value = nodePropertyValues.longArrayValue(i).clone(); + long[] input = nodePropertyValues.longArrayValue(i); + + if (input == null) { + return null; + } + + var value = input.clone(); Arrays.parallelSort(value); return value; }); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java new file mode 100644 index 00000000000..8c9f5a086ab --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.knn.metrics; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.properties.nodes.LongArrayNodePropertyValues; + +import static org.assertj.core.api.Assertions.assertThat; + +class LongArrayPropertySimilarityComputerTest { + + @Test + void usingSparseProperties() { + long[][] inputValues = {{42, 24}, null, {84, 48}}; + + var sparseProperties = new LongArrayNodePropertyValues() { + @Override + public long[] longArrayValue(long nodeId) { + return inputValues[(int) nodeId]; + } + + @Override + public long valuesStored() { + return 2; + } + }; + + var sortedValues = new LongArrayPropertySimilarityComputer.SortedLongArrayPropertyValues(sparseProperties); + + assertThat(sortedValues.longArrayValue(0)).containsExactly(24, 42); + assertThat(sortedValues.longArrayValue(1)).isNull(); + assertThat(sortedValues.longArrayValue(2)).containsExactly(48, 84); + } + +} diff --git a/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java b/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java index 968574f76a4..a8c4ccdda11 100644 --- a/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java +++ b/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java @@ -94,4 +94,32 @@ void shouldStreamWithFilteredNodes() { Map.of("node1", 7L, "node2", 6L, "similarity", 1.0) )); } + + @Test + void computeOverSparseNodeProperties() { + String nodeCreateQuery = + "CREATE " + + " (alice:Person {grades: [24, 4]})" + + " ,(carol:Person)" + + " ,(eve:Person)" + + " ,(bob:Foo {grades: [24, 4, 42]})"; + + runQuery(nodeCreateQuery); + + String createQuery = "CALL gds.graph.project('graph', " + + "'Person', '*', {nodeProperties: {grades: {defaultValue: [1, 1]}}})"; + runQuery(createQuery); + + String algoQuery = GdsCypher.call("graph") + .algo("gds.knn") + .streamMode() + .addParameter("nodeLabels", List.of("Person")) + .addParameter("nodeProperties", List.of("grades")) + .yields("node1", "node2", "similarity"); + + assertCypherResult(algoQuery, List.of( + Map.of("node1", 6L, "node2", 7L, "similarity", 1.0), + Map.of("node1", 7L, "node2", 6L, "similarity", 1.0) + )); + } } From c7a8b15e2cbd42f3fb41391a583c266ae1f7b54e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 2 Mar 2023 16:03:54 +0100 Subject: [PATCH 256/400] Show more wrong usage of valuesStored() This is not exposed to the user though as community properties are not filtered / or the method was wrongly implemented by the node properties --- .../neo4j/gds/CommunityProcCompanionTest.java | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java index 413b5c79b73..a21c44d96fd 100644 --- a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java @@ -25,6 +25,7 @@ import org.neo4j.gds.api.PropertyState; import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.NodeProperty; +import org.neo4j.gds.collections.HugeSparseLongArray; import org.neo4j.gds.config.CommunitySizeConfig; import org.neo4j.gds.config.ConcurrencyConfig; import org.neo4j.gds.config.ConsecutiveIdsConfig; @@ -32,6 +33,9 @@ import org.neo4j.gds.core.CypherMapWrapper; import org.neo4j.gds.nodeproperties.ConsecutiveLongNodePropertyValues; import org.neo4j.gds.nodeproperties.LongIfChangedNodePropertyValues; +import org.neo4j.values.storable.Values; + +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -152,6 +156,60 @@ void shouldWorkWithMinComponentAndConsecutive() { } + @Test + void minComponentSizeWithSparseProperties() { + var inputBuilder = HugeSparseLongArray.builder(Long.MIN_VALUE); + inputBuilder.set(1, 42); + inputBuilder.set(2, 99); + inputBuilder.set(3, 42); + var input = inputBuilder.build(); + + LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, 3, input::get); + + var config = ConfigWithComponentSize.of(CypherMapWrapper.empty().withNumber("minComponentSize", 2L)); + + var filteredProperties = CommunityProcCompanion.nodeProperties( + config, + "seed", + sparseProperties, + () -> { throw new UnsupportedOperationException("Not implemented"); } + ); + + // null from the beginning + assertThat(filteredProperties.value(0)).isNull(); + assertThat(filteredProperties.value(1)).isEqualTo(Values.longValue(42)); + // filtered out + assertThat(filteredProperties.value(2)).isNull(); + assertThat(filteredProperties.value(3)).isEqualTo(Values.longValue(42)); + } + + @Test + void consecutiveIdsWithSparseProperties() { + var inputBuilder = HugeSparseLongArray.builder(Long.MIN_VALUE); + inputBuilder.set(1, 42); + inputBuilder.set(2, 99); + inputBuilder.set(3, 42); + var input = inputBuilder.build(); + + LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, 3, input::get); + + var config = ConfigWithComponentSize.of(CypherMapWrapper.create(Map.of("minComponentSize", 2L, "consecutiveIds", true))); + + var filteredProperties = CommunityProcCompanion.nodeProperties( + config, + "seed", + sparseProperties, + () -> { throw new UnsupportedOperationException("Not implemented"); } + ); + + // null from the beginning + assertThat(filteredProperties.value(0)).isNull(); + assertThat(filteredProperties.value(1)).isEqualTo(Values.longValue(0)); + // filtered out + assertThat(filteredProperties.value(2)).isNull(); + assertThat(filteredProperties.value(3)).isEqualTo(Values.longValue(0)); + } + private static final class TestNodePropertyValues implements LongNodePropertyValues { private final long size; private final LongToLongFunction transformer; @@ -175,6 +233,36 @@ public long longValue(long nodeId) { } } + private static final class TestSparseNodePropertyValues implements LongNodePropertyValues { + private final long size; + private final long valuesStored; + private final LongToLongFunction transformer; + + private TestSparseNodePropertyValues( + long size, + long valuesStored, + LongToLongFunction transformer + ) { + this.size = size; + this.valuesStored = valuesStored; + this.transformer = transformer; + } + + @Override + public long valuesStored() { + return valuesStored; + } + + public long size() { + return size; + } + + @Override + public long longValue(long nodeId) { + return transformer.applyAsLong(nodeId); + } + } + @Configuration interface CommunityProcCompanionConfig extends ConsecutiveIdsConfig, SeedConfig, ConcurrencyConfig, CommunitySizeConfig { static CommunityProcCompanionConfig of(CypherMapWrapper map) { From 28ccd32c085d8340908ba545d228d54d5c058444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 2 Mar 2023 17:41:44 +0100 Subject: [PATCH 257/400] Implement new PropertyValues::maxIndex This can be used to find out the highest index to check if you want to retrieve every value. --- .../neo4j/gds/scaling/ScaleProperties.java | 15 +++++++++++ .../ModularityOptimization.java | 5 ++++ .../gds/pagerank/EigenvectorComputation.java | 5 ++++ .../LongArrayPropertySimilarityComputer.java | 7 +++++- .../NullCheckingNodePropertyValues.java | 5 ++++ .../knn/GenerateRandomNeighborsTest.java | 13 ++-------- ...ngArrayPropertySimilarityComputerTest.java | 5 ++++ .../knn/metrics/SimilarityComputerTest.java | 5 ++++ .../HarmonicCentralityWriteProc.java | 5 ++++ .../CelfNodeProperties.java | 5 ++++ .../gds/scaling/ScalePropertiesProc.java | 9 ++++++- .../java/org/neo4j/gds/scc/SccWriteProc.java | 5 ++++ .../spanningtree/KSpanningTreeWriteSpec.java | 5 ++++ .../gds/api/properties/PropertyValues.java | 6 +++++ .../core/huge/FilteredNodePropertyValues.java | 7 +++++- .../gds/core/loading/NullPropertyMap.java | 10 ++++++++ .../DoubleArrayNodePropertiesBuilder.java | 6 +++++ .../DoubleNodePropertiesBuilder.java | 6 +++++ .../FloatArrayNodePropertiesBuilder.java | 6 +++++ .../LongArrayNodePropertiesBuilder.java | 6 +++++ .../LongNodePropertiesBuilder.java | 6 +++++ .../core/utils/IdentityPropertyValues.java | 5 ++++ .../utils/OriginalIdNodePropertyValues.java | 5 ++++ .../utils/paged/HugeAtomicDoubleArray.java | 5 ++++ .../core/utils/paged/HugeAtomicLongArray.java | 5 ++++ .../gds/core/utils/paged/HugeByteArray.java | 5 ++++ .../gds/core/utils/paged/HugeDoubleArray.java | 5 ++++ .../gds/core/utils/paged/HugeIntArray.java | 5 ++++ .../gds/core/utils/paged/HugeLongArray.java | 5 ++++ .../gds/core/utils/paged/HugeObjectArray.java | 15 +++++++++++ .../utils/paged/dss/DisjointSetStruct.java | 5 ++++ .../gds/core/loading/CSRGraphStoreTest.java | 5 ++++ .../UpdatableDoubleArrayNodeProperty.java | 6 +++++ .../UpdatableDoubleNodeProperty.java | 6 +++++ .../UpdatableFloatArrayNodeProperty.java | 6 +++++ .../UpdatableLongArrayNodeProperty.java | 5 ++++ .../UpdatableLongNodeProperty.java | 5 ++++ .../GraphPropertyStoreFromVisitorHelper.java | 25 +++++++++++++++++++ ...svToGraphStoreImporterIntegrationTest.java | 15 +++++++++++ .../file/csv/GraphStoreToCsvExporterTest.java | 21 +++++++++++++--- .../gds/beta/node2vec/Node2VecCompanion.java | 9 +++++-- .../GraphDropGraphPropertiesProcTest.java | 13 ++++++++-- .../GraphStreamGraphPropertiesProcTest.java | 24 ++++++++++++++++-- .../gds/degree/DegreeCentralityProc.java | 5 ++++ .../ConsecutiveLongNodePropertyValues.java | 19 ++++++++------ ...PropertyComputationResultConsumerTest.java | 18 ++----------- .../neo4j/gds/WriteProcCancellationTest.java | 5 ++++ ...ConsecutiveLongNodePropertyValuesTest.java | 10 +++++--- .../org/neo4j/gds/CommunityProcCompanion.java | 9 +++++-- .../org/neo4j/gds/kmeans/KmeansWriteSpec.java | 11 ++++++-- .../IntermediateCommunityNodeProperties.java | 15 ++++++++--- .../org/neo4j/gds/leiden/LeidenCompanion.java | 1 + .../org/neo4j/gds/louvain/LouvainProc.java | 6 +++++ .../LongIfChangedNodePropertyValues.java | 5 ++++ .../java/org/neo4j/gds/wcc/WccStreamProc.java | 2 +- .../neo4j/gds/CommunityProcCompanionTest.java | 12 ++++++--- .../embeddings/fastrp/FastRPCompanion.java | 5 ++++ .../graphsage/GraphSageCompanion.java | 5 ++++ .../hashgnn/HashGNNProcCompanion.java | 5 ++++ .../NodeClassificationPipelineMutateProc.java | 5 ++++ .../NodeClassificationPipelineWriteProc.java | 5 ++++ .../NodeRegressionPipelineMutateProc.java | 9 +++++-- .../neo4j/gds/pregel/proc/PregelBaseProc.java | 12 +++++++++ .../org/neo4j/gds/test/TestMutateProc.java | 5 ++++ .../DoubleArrayTestPropertyValues.java | 5 ++++ .../DoubleTestPropertyValues.java | 5 ++++ .../FloatArrayTestPropertyValues.java | 5 ++++ .../LongArrayTestPropertyValues.java | 5 ++++ .../LongTestPropertyValues.java | 5 ++++ 69 files changed, 471 insertions(+), 65 deletions(-) diff --git a/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java b/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java index ef8441eb968..c86c7ac59a9 100644 --- a/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java +++ b/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java @@ -228,6 +228,11 @@ public double doubleValue(long nodeId) { public long valuesStored() { return property.valuesStored(); } + + @Override + public long maxIndex() { + return property.maxIndex(); + } }; } @@ -247,6 +252,11 @@ public double doubleValue(long nodeId) { public long valuesStored() { return property.valuesStored(); } + + @Override + public long maxIndex() { + return property.maxIndex(); + } }; } @@ -266,6 +276,11 @@ public double doubleValue(long nodeId) { public long valuesStored() { return property.valuesStored(); } + + @Override + public long maxIndex() { + return property.maxIndex(); + } }; } diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java index b03f52506aa..7e51f0536e4 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java @@ -419,6 +419,11 @@ public long longValue(long nodeId) { public long valuesStored() { return currentCommunities.size(); } + + @Override + public long maxIndex() { + return graph.nodeCount(); + } }; } } diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java b/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java index 5ba0d03b60b..989e57dde3a 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java @@ -119,6 +119,11 @@ public long valuesStored() { return context.nodeCount(); } + @Override + public long maxIndex() { + return context.nodeCount(); + } + @Override public double doubleValue(long nodeId) { return context.doubleNodeValue(nodeId, NEXT_RANK); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java index 61c59cb330b..d8177f9a85c 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java @@ -55,7 +55,7 @@ static final class SortedLongArrayPropertyValues implements LongArrayNodePropert private final HugeObjectArray properties; SortedLongArrayPropertyValues(NodePropertyValues nodePropertyValues) { - this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.valuesStored()); + this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.maxIndex()); this.properties.setAll(i -> { long[] input = nodePropertyValues.longArrayValue(i); @@ -74,6 +74,11 @@ public long valuesStored() { return properties.size(); } + @Override + public long maxIndex() { + return properties.size(); + } + @Override public long[] longArrayValue(long nodeId) { return properties.get(nodeId); diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java index d2c3228dbcf..ead33056c48 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java @@ -90,6 +90,11 @@ public long valuesStored() { return properties.valuesStored(); } + @Override + public long maxIndex() { + return properties.maxIndex(); + } + private void check(long nodeId, @Nullable Object value) { if (value == null) { throw new IllegalArgumentException(formatWithLocale( diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java index 2ed9a853b19..f2f1af4e86d 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java @@ -25,6 +25,7 @@ import org.eclipse.collections.api.tuple.primitive.IntIntPair; import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.core.huge.DirectIdMap; +import org.neo4j.gds.core.utils.IdentityPropertyValues; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.partition.Partition; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -51,17 +52,7 @@ void neighborsForKEqualsNMinus1startWithEachOtherAsNeighbors( nodeCount ); - var nodeProperties = new LongNodePropertyValues() { - @Override - public long longValue(long nodeId) { - return nodeId; - } - - @Override - public long valuesStored() { - return nodeCount; - } - }; + var nodeProperties = new IdentityPropertyValues(nodeCount); var similarityComputer = SimilarityComputer.ofProperty( idMap, diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java index 8c9f5a086ab..4a87a17a972 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java @@ -40,6 +40,11 @@ public long[] longArrayValue(long nodeId) { public long valuesStored() { return 2; } + + @Override + public long maxIndex() { + return 3; + } }; var sortedValues = new LongArrayPropertySimilarityComputer.SortedLongArrayPropertyValues(sparseProperties); diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java index e68a4d8863e..4348c2122a4 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java @@ -266,6 +266,11 @@ public long[] longArrayValue(long nodeId) { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return nodeCount; } }; diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java index 0b331e5aa2b..933991e0e1e 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java @@ -96,6 +96,11 @@ public long valuesStored() { return computationResult.graph().nodeCount(); } + @Override + public long maxIndex() { + return computationResult.graph().nodeCount(); + } + @Override public double doubleValue(long nodeId) { return computationResult.result().getCentralityScore(nodeId); diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/influenceMaximization/CelfNodeProperties.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/influenceMaximization/CelfNodeProperties.java index d1fa54805e6..5f3ad5378e3 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/influenceMaximization/CelfNodeProperties.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/influenceMaximization/CelfNodeProperties.java @@ -34,6 +34,11 @@ class CelfNodeProperties implements DoubleNodePropertyValues { @Override public long valuesStored() { + return celfSeedSet.size(); + } + + @Override + public long maxIndex() { return totalGraphNodeCount; } diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java index dd0d6ca0d11..db81693421d 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/scaling/ScalePropertiesProc.java @@ -23,7 +23,9 @@ import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.executor.ComputationResult; -public class ScalePropertiesProc { +public final class ScalePropertiesProc { + + private ScalePropertiesProc() {} static NodePropertyValues nodeProperties(ComputationResult computationResult) { var size = computationResult.graph().nodeCount(); @@ -32,6 +34,11 @@ static NodePropertyValues nodeProperties(ComputationResult floatArrayValues() { return (Stream) mergeStreamFractions(streamFractions).stream(); @@ -155,6 +170,11 @@ public long valuesStored() { return -1; } + @Override + public long maxIndex() { + return -1; + } + @Override public Stream doubleArrayValues() { return (Stream) mergeStreamFractions(streamFractions).stream(); @@ -174,6 +194,11 @@ public long valuesStored() { return -1; } + @Override + public long maxIndex() { + return -1; + } + @Override public Stream longArrayValues() { return (Stream) mergeStreamFractions(streamFractions).stream(); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index 28a258934ce..555ffc3a2cd 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -169,6 +169,11 @@ public Stream doubleArrayValues() { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 1337; } }); @@ -183,6 +188,11 @@ public LongStream longValues() { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 10_000; } }); @@ -197,6 +207,11 @@ public LongStream longValues() { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 10_000; } }); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java index d4718f9ba22..917bd5efb3e 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java @@ -236,12 +236,17 @@ void shouldExportGraphProperties() { var graphPropertyValues = new LongGraphPropertyValues() { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 3; } @Override public LongStream longValues() { - return LongStream.range(0, valuesStored()); + return LongStream.range(0, maxIndex()); } }; @@ -339,12 +344,17 @@ void exportGraphPropertiesMultithreaded() throws IOException { var graphPropertyValues = new LongGraphPropertyValues() { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 1_000_000; } @Override public LongStream longValues() { - return LongStream.range(0, valuesStored()); + return LongStream.range(0, maxIndex()); } }; @@ -401,11 +411,16 @@ void exportSchemaAndDatabaseId() { graphStore.addGraphProperty("graphProp", new LongGraphPropertyValues() { @Override public LongStream longValues() { - return LongStream.range(0, valuesStored()); + return LongStream.range(0, maxIndex()); } @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 3; } }); diff --git a/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java b/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java index 424eb7ac745..155b643f2db 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java +++ b/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java @@ -35,13 +35,18 @@ final class Node2VecCompanion { static NodePropertyValues nodeProperties( ComputationResult computationResult ) { - var size = computationResult.graph().nodeCount(); + var nodeCount = computationResult.graph().nodeCount(); HugeObjectArray embeddings = computationResult.result().embeddings(); return new FloatArrayNodePropertyValues() { @Override public long valuesStored() { - return size; + return embeddings.size(); + } + + @Override + public long maxIndex() { + return nodeCount; } @Override diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java index d3ac5037215..1854f0ec053 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java @@ -69,8 +69,12 @@ public LongStream longValues() { @Override public long valuesStored() { - return 10; + return 0; + } + @Override + public long maxIndex() { + return 10; } }; @@ -114,9 +118,14 @@ public LongStream longValues() { @Override public long valuesStored() { - return 10; + return 0; } + + @Override + public long maxIndex() { + return 10; + } }; graphStore.addGraphProperty("prop", values); assertError( diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java index 1cc2e1b147b..f8578394386 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java @@ -93,8 +93,13 @@ public LongStream longValues() { @Override public long valuesStored() { - return 10; + return 0; + + } + @Override + public long maxIndex() { + return 10; } }), arguments(new DoubleGraphPropertyValues() { @Override @@ -104,6 +109,11 @@ public DoubleStream doubleValues() { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 3; } }), arguments(new LongArrayGraphPropertyValues() { @@ -114,6 +124,11 @@ public Stream longArrayValues() { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 2; } })); @@ -138,9 +153,14 @@ public LongStream longValues() { @Override public long valuesStored() { - return 10; + return 0; } + + @Override + public long maxIndex() { + return 10; + } }; graphStore.addGraphProperty("prop", values); assertError( diff --git a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java index bb389cb5228..452854f81d7 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java @@ -56,6 +56,11 @@ public long valuesStored() { return size; } + @Override + public long maxIndex() { + return size; + } + @Override public double doubleValue(long nodeId) { return degrees.get(nodeId); diff --git a/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java b/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java index 59dd4fb3f09..0612b3624f0 100644 --- a/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java +++ b/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java @@ -32,22 +32,20 @@ public class ConsecutiveLongNodePropertyValues implements LongNodePropertyValues private final HugeLongArray communities; - public ConsecutiveLongNodePropertyValues( - LongNodePropertyValues longNodeProperties, - long nodeCount - ) { + public ConsecutiveLongNodePropertyValues(LongNodePropertyValues inputProperties) { var nextConsecutiveId = -1L; var nextSkippableId = -2L; // communities signaled with <0 are skipped and not written + long maxIdx = inputProperties.maxIndex(); var setIdToConsecutiveId = new HugeLongLongMap(BitUtil.ceilDiv( - nodeCount, + maxIdx, MAPPING_SIZE_QUOTIENT )); - this.communities = HugeLongArray.newArray(nodeCount); + this.communities = HugeLongArray.newArray(maxIdx); - for (var nodeId = 0; nodeId < nodeCount; nodeId++) { - var setId = longNodeProperties.longValue(nodeId); + for (var nodeId = 0; nodeId < maxIdx; nodeId++) { + var setId = inputProperties.longValue(nodeId); var communityId = setIdToConsecutiveId.getOrDefault(setId, -1); if (communityId == -1) { //if this is null, it means this community should not be written @@ -81,4 +79,9 @@ public Value value(long nodeId) { public long valuesStored() { return communities.size(); } + + @Override + public long maxIndex() { + return communities.size(); + } } diff --git a/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java b/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java index bdfab99798d..98fb531a2f7 100644 --- a/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/MutatePropertyComputationResultConsumerTest.java @@ -27,10 +27,10 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.PropertyState; -import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.api.properties.nodes.NodePropertyValues; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.core.loading.CSRGraphStore; +import org.neo4j.gds.core.utils.IdentityPropertyValues; import org.neo4j.gds.core.utils.progress.EmptyTaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.write.ImmutableNodeProperty; @@ -75,7 +75,7 @@ class MutatePropertyComputationResultConsumerTest { @BeforeEach void setup() { - var nodePropertyList = List.of(ImmutableNodeProperty.of("mutateProperty", new TestNodePropertyValues())); + var nodePropertyList = List.of(ImmutableNodeProperty.of("mutateProperty", new IdentityPropertyValues(graphStore.nodeCount()))); mutateResultConsumer = new MutatePropertyComputationResultConsumer<>( (computationResult) -> nodePropertyList, (computationResult, executionContext) -> new TestAlgoResultBuilder() @@ -161,18 +161,4 @@ private ComputationResult getCom .computeMillis(0) .build(); } - - class TestNodePropertyValues implements LongNodePropertyValues { - - @Override - public long valuesStored() { - return graphStore.nodeCount(); - } - - @Override - public long longValue(long nodeId) { - return nodeId; - } - } - } diff --git a/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java b/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java index b7447feac17..ee13af02708 100644 --- a/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/WriteProcCancellationTest.java @@ -62,6 +62,11 @@ public long longValue(long nodeId) { public long valuesStored() { return 42; } + + @Override + public long maxIndex() { + return 42; + } }; var nodeProperty = ImmutableNodeProperty.of("prop", nodeProperties); diff --git a/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java b/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java index b2bafba8649..9c794855c3f 100644 --- a/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java +++ b/proc/common/src/test/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValuesTest.java @@ -31,6 +31,11 @@ void shouldReturnConsecutiveIds() { LongNodePropertyValues nonConsecutiveIds = new LongNodePropertyValues() { @Override public long valuesStored() { + return 0; + } + + @Override + public long maxIndex() { return 10; } @@ -40,10 +45,7 @@ public long longValue(long nodeId) { } }; - var consecutiveIds = new ConsecutiveLongNodePropertyValues( - nonConsecutiveIds, - 10 - ); + var consecutiveIds = new ConsecutiveLongNodePropertyValues(nonConsecutiveIds); assertThat(consecutiveIds.valuesStored()).isEqualTo(nonConsecutiveIds.valuesStored()); diff --git a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java index 04548ab8e89..10fe69df8af 100644 --- a/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java +++ b/proc/community/src/main/java/org/neo4j/gds/CommunityProcCompanion.java @@ -55,7 +55,7 @@ public static intermediateCommunity; - IntermediateCommunityNodeProperties(long size, LongFunction intermediateCommunity) { - this.size = size; + IntermediateCommunityNodeProperties(long maxIndex, long storedValues, LongFunction intermediateCommunity) { + this.maxIndex = maxIndex; + this.storedValues = storedValues; this.intermediateCommunity = intermediateCommunity; } @Override public long valuesStored() { - return size; + return storedValues; + } + + @Override + public long maxIndex() { + return maxIndex; } @Override diff --git a/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenCompanion.java b/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenCompanion.java index c25a0bafc51..ab38def7ea1 100644 --- a/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenCompanion.java +++ b/proc/community/src/main/java/org/neo4j/gds/leiden/LeidenCompanion.java @@ -38,6 +38,7 @@ static NodePropertyValues leidenNodeProperties if (config.includeIntermediateCommunities()) { return new IntermediateCommunityNodeProperties( + computationResult.graph().nodeCount(), leidenResult.communities().size(), leidenResult::getIntermediateCommunities ); diff --git a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java index ea4948788c0..0efd7d07bd7 100644 --- a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java +++ b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java @@ -55,6 +55,12 @@ static NodePropertyValues nodeProperties( return new LongArrayNodePropertyValues() { @Override public long valuesStored() { + // the result is dense + return size; + } + + @Override + public long maxIndex() { return size; } diff --git a/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java b/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java index 390a310a5ab..efa9133f893 100644 --- a/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java +++ b/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java @@ -79,4 +79,9 @@ public Value value(long nodeId) { public long valuesStored() { return newProperties.valuesStored(); } + + @Override + public long maxIndex() { + return Math.max(newProperties.maxIndex(), seedProperties.maxIndex()); + } } diff --git a/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamProc.java b/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamProc.java index 5c56f6f1ffd..1d94071d536 100644 --- a/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamProc.java +++ b/proc/community/src/main/java/org/neo4j/gds/wcc/WccStreamProc.java @@ -92,7 +92,7 @@ protected NodePropertyValues nodeProperties(ComputationResult DoubleArrayNodePropertyValues getN return new DoubleArrayNodePropertyValues() { @Override public long valuesStored() { + return embeddings.size(); + } + + @Override + public long maxIndex() { return size; } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java index 719f5ae205a..e30b5d5f8d7 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java @@ -41,6 +41,11 @@ public double[] doubleArrayValue(long nodeId) { @Override public long valuesStored() { + return embeddings.size(); + } + + @Override + public long maxIndex() { return nodeCount; } }; diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java index 4149076c1ac..c40859b0cb8 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java @@ -98,6 +98,11 @@ protected List nodePropertyList(ComputationResult nodePropertyList(ComputationResult mutate( @Override protected NodePropertyValues nodeProperties(ComputationResult computationResult) { - var size = computationResult.graph().nodeCount(); + var nodeCount = computationResult.graph().nodeCount(); var predictedPropertyValues = computationResult.result(); return new DoubleNodePropertyValues() { @Override public long valuesStored() { - return size; + return predictedPropertyValues.size(); + } + + @Override + public long maxIndex() { + return nodeCount; } @Override diff --git a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java index f5620d2b4a3..54cde627817 100644 --- a/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java +++ b/proc/pregel/src/main/java/org/neo4j/gds/pregel/proc/PregelBaseProc.java @@ -168,6 +168,12 @@ public long valuesStored() { return longArrays.size(); } + @Override + public long maxIndex() { + // Its backed by a dense array + return longArrays.size(); + } + @Override public long[] longArrayValue(long nodeId) { return longArrays.get(nodeId); @@ -184,6 +190,12 @@ public long valuesStored() { return doubleArrays.size(); } + @Override + public long maxIndex() { + // its backed by dense array + return doubleArrays.size(); + } + @Override public double[] doubleArrayValue(long nodeId) { diff --git a/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java b/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java index 1ba40e5ffe4..8f3dc745863 100644 --- a/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java +++ b/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java @@ -73,6 +73,11 @@ public long longValue(long nodeId) { public long valuesStored() { return 0; } + + @Override + public long maxIndex() { + return 0; + } }; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java index d04d4546946..1cb790df6aa 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java @@ -32,6 +32,11 @@ public long valuesStored() { return 0; } + @Override + public long maxIndex() { + return -1; + } + @Override public double[] doubleArrayValue(long nodeId) { return transformer.apply(nodeId); diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java index 5d97154a815..05430bb253c 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java @@ -33,6 +33,11 @@ public long valuesStored() { return 0; } + @Override + public long maxIndex() { + return 0; + } + @Override public double doubleValue(long nodeId) { return transformer.applyAsDouble(nodeId); diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java index 97a90fff1af..de34291ec3b 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java @@ -32,6 +32,11 @@ public long valuesStored() { return 0; } + @Override + public long maxIndex() { + return -1; + } + @Override public float[] floatArrayValue(long nodeId) { return transformer.apply(nodeId); diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java index 631e0d4ee10..2bb4c80047a 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java @@ -32,6 +32,11 @@ public long valuesStored() { return 0; } + @Override + public long maxIndex() { + return -1; + } + @Override public long[] longArrayValue(long nodeId) { return transformer.apply(nodeId); diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java index 3dc14a8663d..6e3891aecea 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java @@ -32,6 +32,11 @@ public long valuesStored() { return 0; } + @Override + public long maxIndex() { + return -1; + } + @Override public long longValue(long nodeId) { return transformer.applyAsLong(nodeId); From 8c032cb1ec7f131177482e86d64db564e14c2dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 2 Mar 2023 17:49:07 +0100 Subject: [PATCH 258/400] Document sparseness trick in test --- .../org/neo4j/gds/CommunityProcCompanionTest.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java index 4c8a21a7ad0..52498426101 100644 --- a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java @@ -164,6 +164,7 @@ void minComponentSizeWithSparseProperties() { inputBuilder.set(3, 42); var input = inputBuilder.build(); + // we mimic the sparseness here through size > valueStored LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, 3, input::get); var config = ConfigWithComponentSize.of(CypherMapWrapper.empty().withNumber("minComponentSize", 2L)); @@ -175,10 +176,8 @@ void minComponentSizeWithSparseProperties() { () -> { throw new UnsupportedOperationException("Not implemented"); } ); - // null from the beginning assertThat(filteredProperties.value(0)).isNull(); assertThat(filteredProperties.value(1)).isEqualTo(Values.longValue(42)); - // filtered out assertThat(filteredProperties.value(2)).isNull(); assertThat(filteredProperties.value(3)).isEqualTo(Values.longValue(42)); } @@ -191,6 +190,7 @@ void consecutiveIdsWithSparseProperties() { inputBuilder.set(3, 42); var input = inputBuilder.build(); + // we mimic the sparseness here through size > valueStored LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, 3, input::get); var config = ConfigWithComponentSize.of(CypherMapWrapper.create(Map.of("consecutiveIds", true))); @@ -202,12 +202,10 @@ void consecutiveIdsWithSparseProperties() { () -> { throw new UnsupportedOperationException("Not implemented"); } ); - // null from the beginning - assertThat(filteredProperties.value(0)).isNull(); - assertThat(filteredProperties.value(1)).isEqualTo(Values.longValue(0)); - // filtered out - assertThat(filteredProperties.value(2)).isEqualTo(Values.longValue(1)); - assertThat(filteredProperties.value(3)).isEqualTo(Values.longValue(0)); + assertThat(filteredProperties.value(0)).isEqualTo(Values.longValue(0)); + assertThat(filteredProperties.value(1)).isEqualTo(Values.longValue(1)); + assertThat(filteredProperties.value(2)).isEqualTo(Values.longValue(2)); + assertThat(filteredProperties.value(3)).isEqualTo(Values.longValue(1)); } private static final class TestNodePropertyValues implements LongNodePropertyValues { From 576fae21a9017868919a0041aa22aa207506b387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 6 Mar 2023 15:32:41 +0100 Subject: [PATCH 259/400] Rename maxIdx nodeCount/valueCount Also replace all usage of `valuesStored` --- .../neo4j/gds/scaling/ScaleProperties.java | 27 ++++---------- .../ModularityOptimization.java | 10 ++---- .../gds/pagerank/EigenvectorComputation.java | 7 +--- .../LongArrayPropertySimilarityComputer.java | 9 ++--- .../NullCheckingNodePropertyValues.java | 9 ++--- ...ngArrayPropertySimilarityComputerTest.java | 9 ++--- .../knn/metrics/SimilarityComputerTest.java | 7 +--- .../HarmonicCentralityWriteProc.java | 7 +--- .../CelfNodeProperties.java | 7 +--- .../gds/scaling/ScalePropertiesProc.java | 7 +--- .../java/org/neo4j/gds/scc/SccWriteProc.java | 7 +--- .../spanningtree/KSpanningTreeWriteSpec.java | 7 +--- .../gds/api/properties/PropertyValues.java | 11 ------ .../properties/graph/GraphPropertyValues.java | 2 ++ .../nodes/DoubleNodePropertyValues.java | 2 +- .../nodes/LongNodePropertyValues.java | 2 +- .../properties/nodes/NodePropertyValues.java | 2 ++ .../core/huge/FilteredNodePropertyValues.java | 8 +---- .../gds/core/loading/NullPropertyMap.java | 14 ++------ .../core/loading/ScanningNodesImporter.java | 2 +- .../DoubleArrayNodePropertiesBuilder.java | 8 +---- .../DoubleNodePropertiesBuilder.java | 8 +---- .../FloatArrayNodePropertiesBuilder.java | 8 +---- .../LongArrayNodePropertiesBuilder.java | 8 +---- .../LongNodePropertiesBuilder.java | 8 +---- .../core/utils/IdentityPropertyValues.java | 7 +--- .../utils/OriginalIdNodePropertyValues.java | 7 +--- .../utils/paged/HugeAtomicDoubleArray.java | 7 +--- .../core/utils/paged/HugeAtomicLongArray.java | 7 +--- .../gds/core/utils/paged/HugeByteArray.java | 7 +--- .../gds/core/utils/paged/HugeDoubleArray.java | 7 +--- .../gds/core/utils/paged/HugeIntArray.java | 7 +--- .../gds/core/utils/paged/HugeLongArray.java | 7 +--- .../gds/core/utils/paged/HugeObjectArray.java | 21 ++--------- .../utils/paged/dss/DisjointSetStruct.java | 7 +--- .../gds/core/loading/CSRGraphStoreTest.java | 7 +--- .../core/loading/HugeGraphLoadingTest.java | 4 +-- .../NodePropertiesFromStoreBuilderTest.java | 8 ++--- .../core/utils/paged/HugeObjectArrayTest.java | 6 ++-- .../UpdatableDoubleArrayNodeProperty.java | 8 +---- .../UpdatableDoubleNodeProperty.java | 9 +---- .../UpdatableFloatArrayNodeProperty.java | 8 +---- .../UpdatableLongArrayNodeProperty.java | 7 +--- .../UpdatableLongNodeProperty.java | 8 +---- .../GraphPropertyStoreFromVisitorHelper.java | 35 +++---------------- ...svToGraphStoreImporterIntegrationTest.java | 21 ++--------- .../file/csv/GraphStoreToCsvExporterTest.java | 28 ++++----------- .../gds/beta/node2vec/Node2VecCompanion.java | 7 +--- .../ModularityOptimizationProc.java | 2 +- .../catalog/GraphDropGraphPropertiesProc.java | 2 +- .../catalog/GraphDropNodePropertiesProc.java | 2 +- .../GraphDropGraphPropertiesProcTest.java | 15 ++------ .../GraphStreamGraphPropertiesProcTest.java | 32 +++-------------- .../gds/degree/DegreeCentralityProc.java | 6 +--- .../ConsecutiveLongNodePropertyValues.java | 9 ++--- .../neo4j/gds/WriteProcCancellationTest.java | 7 +--- ...ConsecutiveLongNodePropertyValuesTest.java | 9 ++--- .../org/neo4j/gds/CommunityProcCompanion.java | 11 ++---- .../org/neo4j/gds/kmeans/KmeansWriteSpec.java | 7 +--- .../IntermediateCommunityNodeProperties.java | 15 +++----- .../org/neo4j/gds/louvain/LouvainProc.java | 8 +---- .../LongIfChangedNodePropertyValues.java | 9 ++--- .../neo4j/gds/CommunityProcCompanionTest.java | 25 ++++--------- .../embeddings/fastrp/FastRPCompanion.java | 7 +--- .../graphsage/GraphSageCompanion.java | 7 +--- .../hashgnn/HashGNNProcCompanion.java | 7 +--- .../NodeClassificationPipelineMutateProc.java | 7 +--- .../NodeClassificationPipelineWriteProc.java | 7 +--- .../NodeRegressionPipelineMutateProc.java | 7 +--- ...eClassificationPipelineMutateProcTest.java | 6 ++-- .../NodeRegressionPipelineMutateProcTest.java | 2 +- .../neo4j/gds/pregel/proc/PregelBaseProc.java | 14 ++------ .../org/neo4j/gds/test/TestMutateProc.java | 7 +--- .../DoubleArrayTestPropertyValues.java | 7 +--- .../DoubleTestPropertyValues.java | 7 +--- .../FloatArrayTestPropertyValues.java | 7 +--- .../LongArrayTestPropertyValues.java | 7 +--- .../LongTestPropertyValues.java | 7 +--- 78 files changed, 131 insertions(+), 570 deletions(-) diff --git a/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java b/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java index c86c7ac59a9..bd540bedb79 100644 --- a/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java +++ b/algo-common/src/main/java/org/neo4j/gds/scaling/ScaleProperties.java @@ -225,13 +225,8 @@ public double doubleValue(long nodeId) { } @Override - public long valuesStored() { - return property.valuesStored(); - } - - @Override - public long maxIndex() { - return property.maxIndex(); + public long nodeCount() { + return property.nodeCount(); } }; } @@ -249,13 +244,8 @@ public double doubleValue(long nodeId) { } @Override - public long valuesStored() { - return property.valuesStored(); - } - - @Override - public long maxIndex() { - return property.maxIndex(); + public long nodeCount() { + return property.nodeCount(); } }; } @@ -273,13 +263,8 @@ public double doubleValue(long nodeId) { } @Override - public long valuesStored() { - return property.valuesStored(); - } - - @Override - public long maxIndex() { - return property.maxIndex(); + public long nodeCount() { + return property.nodeCount(); } }; } diff --git a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java index 7e51f0536e4..546b16818ee 100644 --- a/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java +++ b/algo/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimization.java @@ -414,14 +414,8 @@ public LongNodePropertyValues asNodeProperties() { public long longValue(long nodeId) { return getCommunityId(nodeId); } - - @Override - public long valuesStored() { - return currentCommunities.size(); - } - - @Override - public long maxIndex() { + @Override + public long nodeCount() { return graph.nodeCount(); } }; diff --git a/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java b/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java index 989e57dde3a..be38380605f 100644 --- a/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java +++ b/algo/src/main/java/org/neo4j/gds/pagerank/EigenvectorComputation.java @@ -115,12 +115,7 @@ public boolean masterCompute(MasterComputeContext context) { var properties = new DoubleNodePropertyValues() { @Override - public long valuesStored() { - return context.nodeCount(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return context.nodeCount(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java index d8177f9a85c..5295efeab76 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputer.java @@ -55,7 +55,7 @@ static final class SortedLongArrayPropertyValues implements LongArrayNodePropert private final HugeObjectArray properties; SortedLongArrayPropertyValues(NodePropertyValues nodePropertyValues) { - this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.maxIndex()); + this.properties = HugeObjectArray.newArray(long[].class, nodePropertyValues.nodeCount()); this.properties.setAll(i -> { long[] input = nodePropertyValues.longArrayValue(i); @@ -70,12 +70,7 @@ static final class SortedLongArrayPropertyValues implements LongArrayNodePropert } @Override - public long valuesStored() { - return properties.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return properties.size(); } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java index ead33056c48..c1e37ddd4e7 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/knn/metrics/NullCheckingNodePropertyValues.java @@ -86,13 +86,8 @@ public Value value(long nodeId) { } @Override - public long valuesStored() { - return properties.valuesStored(); - } - - @Override - public long maxIndex() { - return properties.maxIndex(); + public long nodeCount() { + return properties.nodeCount(); } private void check(long nodeId, @Nullable Object value) { diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java index 4a87a17a972..4c2bfe93969 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/LongArrayPropertySimilarityComputerTest.java @@ -37,13 +37,8 @@ public long[] longArrayValue(long nodeId) { } @Override - public long valuesStored() { - return 2; - } - - @Override - public long maxIndex() { - return 3; + public long nodeCount() { + return inputValues.length; } }; diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java index 4348c2122a4..33359d3a7a6 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/metrics/SimilarityComputerTest.java @@ -265,12 +265,7 @@ public long[] longArrayValue(long nodeId) { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } }; diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java index 933991e0e1e..dc8ec4e7680 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/centrality/HarmonicCentralityWriteProc.java @@ -92,12 +92,7 @@ public ComputationResultConsumer longArrayValues() { Stream objects(); Stream values(); + + long valueCount(); } diff --git a/core/src/main/java/org/neo4j/gds/api/properties/nodes/DoubleNodePropertyValues.java b/core/src/main/java/org/neo4j/gds/api/properties/nodes/DoubleNodePropertyValues.java index 30f414b5167..d61824abfd4 100644 --- a/core/src/main/java/org/neo4j/gds/api/properties/nodes/DoubleNodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/api/properties/nodes/DoubleNodePropertyValues.java @@ -50,7 +50,7 @@ default ValueType valueType() { @Override default OptionalDouble getMaxDoublePropertyValue() { return LongStream - .range(0, size()) + .range(0, nodeCount()) .parallel() .mapToDouble(this::doubleValue) .max(); diff --git a/core/src/main/java/org/neo4j/gds/api/properties/nodes/LongNodePropertyValues.java b/core/src/main/java/org/neo4j/gds/api/properties/nodes/LongNodePropertyValues.java index 514e3e2a935..9d1ff7b9e37 100644 --- a/core/src/main/java/org/neo4j/gds/api/properties/nodes/LongNodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/api/properties/nodes/LongNodePropertyValues.java @@ -60,7 +60,7 @@ default double doubleValue(long nodeId) { @Override default OptionalLong getMaxLongPropertyValue() { return LongStream - .range(0, valuesStored()) + .range(0, nodeCount()) .parallel() .map(this::longValue) .max(); diff --git a/core/src/main/java/org/neo4j/gds/api/properties/nodes/NodePropertyValues.java b/core/src/main/java/org/neo4j/gds/api/properties/nodes/NodePropertyValues.java index 93c9c3778d0..5d89a9f0130 100644 --- a/core/src/main/java/org/neo4j/gds/api/properties/nodes/NodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/api/properties/nodes/NodePropertyValues.java @@ -59,6 +59,8 @@ default long[] longArrayValue(long nodeId) { Value value(long nodeId); + long nodeCount(); + /** * @return the maximum long value contained in the mapping or an empty {@link OptionalLong} if the mapping is * empty or the feature is not supported. diff --git a/core/src/main/java/org/neo4j/gds/core/huge/FilteredNodePropertyValues.java b/core/src/main/java/org/neo4j/gds/core/huge/FilteredNodePropertyValues.java index 4654820665f..1be2bdec92e 100644 --- a/core/src/main/java/org/neo4j/gds/core/huge/FilteredNodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/huge/FilteredNodePropertyValues.java @@ -141,14 +141,8 @@ public long release() { graph = null; return releasedFromProps; } - - @Override - public long valuesStored() { - return properties.valuesStored(); - } - @Override - public long maxIndex() { + public long nodeCount() { return graph.nodeCount(); } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NullPropertyMap.java b/core/src/main/java/org/neo4j/gds/core/loading/NullPropertyMap.java index b4edbd36263..dbf35141ef7 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NullPropertyMap.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NullPropertyMap.java @@ -65,12 +65,7 @@ public OptionalDouble getMaxDoublePropertyValue() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return 0; } } @@ -106,12 +101,7 @@ public ValueType valueType() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return 0; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java index 923c22c08b7..93d3a3863d6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ScanningNodesImporter.java @@ -265,7 +265,7 @@ private void importPropertiesFromIndex( for (var entry : buildersByPropertyKey.entrySet()) { NodePropertyValues propertyValues = entry.getValue().build(idMap); nodeProperties.put(entry.getKey(), propertyValues); - recordsImported += propertyValues.valuesStored(); + recordsImported += propertyValues.nodeCount(); } long tookNanos = System.nanoTime() - indexStart; diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java index d02aee700a1..1bb7ac3fbbb 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleArrayNodePropertiesBuilder.java @@ -116,13 +116,7 @@ public double[] doubleArrayValue(long nodeId) { } @Override - public long valuesStored() { - // FIXME does not reflect the sparse array - return propertyValues.capacity(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java index 072edfaafc4..f955721dfb1 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java @@ -183,13 +183,7 @@ public OptionalDouble getMaxDoublePropertyValue() { } @Override - public long valuesStored() { - // At the moment, this is reporting the upperbound of values stored. - return propertyValues.capacity(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java index 05186db566b..86633552e96 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/FloatArrayNodePropertiesBuilder.java @@ -114,13 +114,7 @@ public float[] floatArrayValue(long nodeId) { } @Override - public long valuesStored() { - // FIXME this reports not the number of stored values - return propertyValues.capacity(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java index 59327e17463..6c3616b641a 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongArrayNodePropertiesBuilder.java @@ -119,13 +119,7 @@ public long[] longArrayValue(long nodeId) { } @Override - public long valuesStored() { - // FIXME this does not actually contain the number of stored values in the sparse array - return size; - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java index 1e87a9a7dbb..c33877761a3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java @@ -191,13 +191,7 @@ public OptionalLong getMaxLongPropertyValue() { } @Override - public long valuesStored() { - // FIXME this does not reflect the sparse array - return propertyValues.capacity(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/IdentityPropertyValues.java b/core/src/main/java/org/neo4j/gds/core/utils/IdentityPropertyValues.java index 370a8cde336..1f0d0af645a 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/IdentityPropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/IdentityPropertyValues.java @@ -34,12 +34,7 @@ public long longValue(long nodeId) { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return expectedPropertyCount; } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/OriginalIdNodePropertyValues.java b/core/src/main/java/org/neo4j/gds/core/utils/OriginalIdNodePropertyValues.java index 8f329ae83a9..afcd5d5806a 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/OriginalIdNodePropertyValues.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/OriginalIdNodePropertyValues.java @@ -42,12 +42,7 @@ public OptionalLong getMaxLongPropertyValue() { } @Override - public long valuesStored() { - return idMap.nodeCount(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return idMap.nodeCount(); } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicDoubleArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicDoubleArray.java index 384177f701b..922165f2993 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicDoubleArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicDoubleArray.java @@ -175,14 +175,9 @@ public double doubleValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeAtomicDoubleArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicLongArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicLongArray.java index 8e7498fc884..e79b4daae0a 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicLongArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeAtomicLongArray.java @@ -204,14 +204,9 @@ public long longValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeAtomicLongArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeByteArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeByteArray.java index 8b832beb9ce..4faefaba1b0 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeByteArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeByteArray.java @@ -205,14 +205,9 @@ public long longValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeByteArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeDoubleArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeDoubleArray.java index f868f3adc1b..b0c228fd89e 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeDoubleArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeDoubleArray.java @@ -189,14 +189,9 @@ public double doubleValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeDoubleArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeIntArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeIntArray.java index 6290f96b378..b029ae82c22 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeIntArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeIntArray.java @@ -205,14 +205,9 @@ public long longValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeIntArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeLongArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeLongArray.java index d201e640bb8..6806830798c 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeLongArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeLongArray.java @@ -207,14 +207,9 @@ public long longValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeLongArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeObjectArray.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeObjectArray.java index e043baea46f..b4b1fb3fc41 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeObjectArray.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/HugeObjectArray.java @@ -190,14 +190,9 @@ public float[] floatArrayValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeObjectArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } if (cls == double[].class) { @@ -208,14 +203,9 @@ public double[] doubleArrayValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeObjectArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } if (cls == long[].class) { @@ -226,14 +216,9 @@ public long[] longArrayValue(long nodeId) { } @Override - public long valuesStored() { + public long nodeCount() { return HugeObjectArray.this.size(); } - - @Override - public long maxIndex() { - return valuesStored(); - } }; } throw new UnsupportedOperationException("This HugeObjectArray can not be converted to node properties."); diff --git a/core/src/main/java/org/neo4j/gds/core/utils/paged/dss/DisjointSetStruct.java b/core/src/main/java/org/neo4j/gds/core/utils/paged/dss/DisjointSetStruct.java index e486efcd94e..11ae9672912 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/paged/dss/DisjointSetStruct.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/paged/dss/DisjointSetStruct.java @@ -77,12 +77,7 @@ public long longValue(long nodeId) { } @Override - public long valuesStored() { - return DisjointSetStruct.this.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return DisjointSetStruct.this.size(); } }; diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java index cdd6ad422d5..6aff8ee9b32 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CSRGraphStoreTest.java @@ -253,12 +253,7 @@ public LongStream longValues() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long valueCount() { return 4; } }); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/HugeGraphLoadingTest.java b/core/src/test/java/org/neo4j/gds/core/loading/HugeGraphLoadingTest.java index 1ba62e7930e..0e2af30bf08 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/HugeGraphLoadingTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/HugeGraphLoadingTest.java @@ -105,10 +105,10 @@ private void testPropertyLoading(int maxArrayLengthShift) { .graph(); NodePropertyValues nodePropertyValues = graph.nodeProperties("bar"); - long propertyCountDiff = nodeCount - nodePropertyValues.valuesStored(); + long propertyCountDiff = nodeCount - nodePropertyValues.nodeCount(); String errorMessage = formatWithLocale( "Expected %d properties to be imported. Actually imported %d properties (missing %d properties).", - nodeCount, nodePropertyValues.valuesStored(), propertyCountDiff + nodeCount, nodePropertyValues.nodeCount(), propertyCountDiff ); assertEquals(0, propertyCountDiff, errorMessage); diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java index b438ad73820..7d3e02286cb 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java @@ -61,7 +61,7 @@ void testEmptyDoubleProperties() { 1 ).build(idMap(nodeCount)); - assertEquals(0L, properties.valuesStored()); + assertEquals(0L, properties.nodeCount()); assertEquals(OptionalDouble.empty(), properties.getMaxDoublePropertyValue()); assertEquals(42.0, properties.doubleValue(0)); } @@ -74,7 +74,7 @@ void testEmptyLongProperties() { 1 ).build(idMap(nodeCount)); - assertEquals(0L, properties.valuesStored()); + assertEquals(0L, properties.nodeCount()); assertEquals(OptionalLong.empty(), properties.getMaxLongPropertyValue()); assertEquals(42, properties.longValue(0)); } @@ -241,7 +241,7 @@ void hasSize() { b.set(0, Values.of(42.0)); b.set(1, Values.of(21.0)); }); - assertEquals(2, properties.valuesStored()); + assertEquals(2, properties.nodeCount()); } @Test @@ -304,7 +304,7 @@ void threadSafety() throws InterruptedException { var expected = i == 1338 ? 0x1p41 : i == 1337 ? 0x1p42 : i % 2 == 0 ? 2.0 : 1.0; assertEquals(expected, properties.doubleValue(i), "" + i); } - assertEquals(nodeSize, properties.valuesStored()); + assertEquals(nodeSize, properties.nodeCount()); var maxPropertyValue = properties.getMaxDoublePropertyValue(); assertTrue(maxPropertyValue.isPresent()); diff --git a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeObjectArrayTest.java b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeObjectArrayTest.java index 793eeafbb2d..88013671d69 100644 --- a/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeObjectArrayTest.java +++ b/core/src/test/java/org/neo4j/gds/core/utils/paged/HugeObjectArrayTest.java @@ -53,7 +53,7 @@ void shouldReturnNodePropertiesForFloatArrayValues(HugeObjectArray floa assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(FloatArrayNodePropertyValues.class)) .describedAs("float properties must return the same size and elements as the underlying array") - .returns(floats.size(), FloatArrayNodePropertyValues::valuesStored) + .returns(floats.size(), FloatArrayNodePropertyValues::nodeCount) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.floatArrayValue(nodeId)) @@ -75,7 +75,7 @@ void shouldReturnNodePropertiesForDoubleArrayValues(HugeObjectArray do assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(DoubleArrayNodePropertyValues.class)) .describedAs("double properties must return the same size and elements as the underlying array") - .returns(doubles.size(), DoubleArrayNodePropertyValues::valuesStored) + .returns(doubles.size(), DoubleArrayNodePropertyValues::nodeCount) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.doubleArrayValue(nodeId)) @@ -104,7 +104,7 @@ void shouldReturnNodePropertiesForLongArrayValues(HugeObjectArray longs) assertThat(nodeProperties) .asInstanceOf(InstanceOfAssertFactories.type(LongArrayNodePropertyValues.class)) .describedAs("long properties must return the same size and elements as the underlying array") - .returns(longs.size(), LongArrayNodePropertyValues::valuesStored) + .returns(longs.size(), LongArrayNodePropertyValues::nodeCount) .satisfies(array -> { for (int nodeId = 0; nodeId < NODE_COUNT; nodeId++) { assertThat(array.longArrayValue(nodeId)) diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java index bc6264896d1..436bab6510e 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleArrayNodeProperty.java @@ -36,13 +36,7 @@ public UpdatableDoubleArrayNodeProperty(long nodeCount, double[] defaultValue) { } @Override - public long valuesStored() { - // FIXME this does not reflect the number of values stored - return doubleArrayList.capacity(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java index a45900a4197..5534a4f8438 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableDoubleNodeProperty.java @@ -35,15 +35,8 @@ public UpdatableDoubleNodeProperty(long nodeCount, double defaultValue) { this.doubleList = HugeSparseDoubleList.of(defaultValue); } - - @Override - public long valuesStored() { - // FIXME does not reflect sparse array - return doubleList.capacity(); - } - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java index c995c89de6e..6075249a460 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableFloatArrayNodeProperty.java @@ -36,13 +36,7 @@ public UpdatableFloatArrayNodeProperty(long nodeCount, float[] defaultValue) { } @Override - public long valuesStored() { - // FIXME does not reflect sparse array - return floatArrayList.capacity(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java index 6201669ee04..7ae269f5a64 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongArrayNodeProperty.java @@ -36,12 +36,7 @@ public UpdatableLongArrayNodeProperty(long nodeCount, long[] defaultValue) { } @Override - public long valuesStored() { - return longArrayList.capacity(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java index 375dbc3b430..b64868d265e 100644 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java +++ b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/nodeproperties/UpdatableLongNodeProperty.java @@ -35,14 +35,8 @@ public UpdatableLongNodeProperty(long nodeCount, long defaultValue) { this.longList = HugeSparseLongList.of(defaultValue); } - - @Override - public long valuesStored() { - return longList.capacity(); - } - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java index c81512d58d2..506e000df11 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/GraphPropertyStoreFromVisitorHelper.java @@ -93,12 +93,7 @@ static class LongStreamBuilderGraphPropertyValues implements LongGraphPropertyVa } @Override - public long valuesStored() { - return -1; - } - - @Override - public long maxIndex() { + public long valueCount() { return -1; } @@ -117,12 +112,7 @@ static class DoubleStreamBuilderGraphPropertyValues implements DoubleGraphProper } @Override - public long valuesStored() { - return -1; - } - - @Override - public long maxIndex() { + public long valueCount() { return -1; } @@ -142,12 +132,7 @@ static class FloatArrayStreamBuilderGraphPropertyValues implements FloatArrayGra } @Override - public long valuesStored() { - return -1; - } - - @Override - public long maxIndex() { + public long valueCount() { return -1; } @@ -166,12 +151,7 @@ static class DoubleArrayStreamBuilderGraphPropertyValues implements DoubleArrayG } @Override - public long valuesStored() { - return -1; - } - - @Override - public long maxIndex() { + public long valueCount() { return -1; } @@ -190,12 +170,7 @@ static class LongArrayStreamBuilderGraphPropertyValues implements LongArrayGraph } @Override - public long valuesStored() { - return -1; - } - - @Override - public long maxIndex() { + public long valueCount() { return -1; } diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java index 555ffc3a2cd..d85d7f397f2 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/CsvToGraphStoreImporterIntegrationTest.java @@ -168,12 +168,7 @@ public Stream doubleArrayValues() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long valueCount() { return 1337; } }); @@ -187,12 +182,7 @@ public LongStream longValues() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long valueCount() { return 10_000; } }); @@ -206,12 +196,7 @@ public LongStream longValues() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long valueCount() { return 10_000; } }); diff --git a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java index 917bd5efb3e..54a4ed41e59 100644 --- a/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java +++ b/io/csv/src/test/java/org/neo4j/gds/core/io/file/csv/GraphStoreToCsvExporterTest.java @@ -235,18 +235,13 @@ void shouldExportGraphProperties() { var graphPropertyValues = new LongGraphPropertyValues() { @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long valueCount() { return 3; } @Override public LongStream longValues() { - return LongStream.range(0, maxIndex()); + return LongStream.range(0, valueCount()); } }; @@ -342,19 +337,15 @@ void exportGraphPropertiesMultithreaded() throws IOException { .build(); var graphPropertyValues = new LongGraphPropertyValues() { - @Override - public long valuesStored() { - return 0; - } @Override - public long maxIndex() { + public long valueCount() { return 1_000_000; } @Override public LongStream longValues() { - return LongStream.range(0, maxIndex()); + return LongStream.range(0, valueCount()); } }; @@ -396,7 +387,7 @@ public LongStream longValues() { .sorted() .toArray(); - assertArrayEquals(LongStream.range(0, graphPropertyValues.valuesStored()).toArray(), exportedValues); + assertArrayEquals(LongStream.range(0, graphPropertyValues.valueCount()).toArray(), exportedValues); } @Test @@ -411,16 +402,11 @@ void exportSchemaAndDatabaseId() { graphStore.addGraphProperty("graphProp", new LongGraphPropertyValues() { @Override public LongStream longValues() { - return LongStream.range(0, maxIndex()); - } - - @Override - public long valuesStored() { - return 0; + return LongStream.range(0, valueCount()); } @Override - public long maxIndex() { + public long valueCount() { return 3; } }); diff --git a/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java b/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java index 155b643f2db..0dbd8f0c4a9 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java +++ b/proc/beta/src/main/java/org/neo4j/gds/beta/node2vec/Node2VecCompanion.java @@ -40,12 +40,7 @@ static NodePropertyValues nodeProperties( return new FloatArrayNodePropertyValues() { @Override - public long valuesStored() { - return embeddings.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } diff --git a/proc/beta/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationProc.java b/proc/beta/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationProc.java index 5b9e3d73f0f..89d442aa43c 100644 --- a/proc/beta/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationProc.java +++ b/proc/beta/src/main/java/org/neo4j/gds/modularityoptimization/ModularityOptimizationProc.java @@ -50,7 +50,7 @@ static NodePropertyValues nodeProp ) { LongNodePropertyValues resultCommunities = computationResult.result().asNodeProperties(); if (computationResult.config().consecutiveIds()) { - return new ConsecutiveLongNodePropertyValues(resultCommunities, computationResult.graph().nodeCount()); + return new ConsecutiveLongNodePropertyValues(resultCommunities); } else { return resultCommunities; } diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java index 61ceffbe9fa..05770540154 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProc.java @@ -57,7 +57,7 @@ public Stream run( config.validate(graphStore); // removing - long propertiesRemoved = graphStore.graphPropertyValues(graphProperty).valuesStored(); + long propertiesRemoved = graphStore.graphPropertyValues(graphProperty).valueCount(); runWithExceptionLogging( "Graph property removal failed", () -> graphStore.removeGraphProperty(graphProperty) diff --git a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java index 9c2a54e5804..4d4210ff286 100644 --- a/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java +++ b/proc/catalog/src/main/java/org/neo4j/gds/catalog/GraphDropNodePropertiesProc.java @@ -120,7 +120,7 @@ private Long dropNodeProperties( progressTracker.beginSubTask(); config.nodeProperties().forEach(propertyKey -> { - removedPropertiesCount.add(graphStore.nodeProperty(propertyKey).values().valuesStored()); + removedPropertiesCount.add(graphStore.nodeProperty(propertyKey).values().nodeCount()); graphStore.removeNodeProperty(propertyKey); progressTracker.logProgress(); }); diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java index 1854f0ec053..65140877e6e 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphDropGraphPropertiesProcTest.java @@ -68,12 +68,7 @@ public LongStream longValues() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long valueCount() { return 10; } }; @@ -117,13 +112,7 @@ public LongStream longValues() { } @Override - public long valuesStored() { - return 0; - - } - - @Override - public long maxIndex() { + public long valueCount() { return 10; } }; diff --git a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java index f8578394386..a4c67dc1555 100644 --- a/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java +++ b/proc/catalog/src/test/java/org/neo4j/gds/catalog/GraphStreamGraphPropertiesProcTest.java @@ -92,13 +92,7 @@ public LongStream longValues() { } @Override - public long valuesStored() { - return 0; - - } - - @Override - public long maxIndex() { + public long valueCount() { return 10; } }), arguments(new DoubleGraphPropertyValues() { @@ -108,12 +102,7 @@ public DoubleStream doubleValues() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long valueCount() { return 3; } }), arguments(new LongArrayGraphPropertyValues() { @@ -123,12 +112,7 @@ public Stream longArrayValues() { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long valueCount() { return 2; } })); @@ -148,17 +132,11 @@ void shouldFailOnNonExistingNodePropertyForSpecificLabel() { LongGraphPropertyValues values = new LongGraphPropertyValues() { @Override public LongStream longValues() { - return LongStream.range(0, 10); - } - - @Override - public long valuesStored() { - return 0; - + return LongStream.range(0, valueCount()); } @Override - public long maxIndex() { + public long valueCount() { return 10; } }; diff --git a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java index 452854f81d7..7e861aaffc2 100644 --- a/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java +++ b/proc/centrality/src/main/java/org/neo4j/gds/degree/DegreeCentralityProc.java @@ -51,13 +51,9 @@ static NodePropertyValues nodeProperties(ComputationResult intermediateCommunity; - IntermediateCommunityNodeProperties(long maxIndex, long storedValues, LongFunction intermediateCommunity) { - this.maxIndex = maxIndex; + IntermediateCommunityNodeProperties(long nodeCount, long storedValues, LongFunction intermediateCommunity) { + this.nodeCount = nodeCount; this.storedValues = storedValues; this.intermediateCommunity = intermediateCommunity; } @Override - public long valuesStored() { - return storedValues; - } - - @Override - public long maxIndex() { - return maxIndex; + public long nodeCount() { + return nodeCount; } @Override diff --git a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java index 0efd7d07bd7..cc4af49e6bb 100644 --- a/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java +++ b/proc/community/src/main/java/org/neo4j/gds/louvain/LouvainProc.java @@ -54,13 +54,7 @@ static NodePropertyValues nodeProperties( return new LongArrayNodePropertyValues() { @Override - public long valuesStored() { - // the result is dense - return size; - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } diff --git a/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java b/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java index efa9133f893..5500182b4cb 100644 --- a/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java +++ b/proc/community/src/main/java/org/neo4j/gds/nodeproperties/LongIfChangedNodePropertyValues.java @@ -76,12 +76,7 @@ public Value value(long nodeId) { } @Override - public long valuesStored() { - return newProperties.valuesStored(); - } - - @Override - public long maxIndex() { - return Math.max(newProperties.maxIndex(), seedProperties.maxIndex()); + public long nodeCount() { + return Math.max(newProperties.nodeCount(), seedProperties.nodeCount()); } } diff --git a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java index 52498426101..ca6633ff3dc 100644 --- a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java @@ -91,7 +91,7 @@ void shouldReturnOnlyChangedProperties() { ); assertThat(result).isInstanceOf(LongIfChangedNodePropertyValues.class); - for (long i = 0; i < result.valuesStored(); i++) { + for (long i = 0; i < result.nodeCount(); i++) { assertThat(result.longValue(i)).isEqualTo(inputProperties.longValue(i)); assertThat(result.value(i)).isNull(); } @@ -110,7 +110,7 @@ void shouldRestrictCommunitySize() { () -> { throw new UnsupportedOperationException("Not implemented"); } ); - for (long i = 0L; i < result.valuesStored(); i++) { + for (long i = 0L; i < result.nodeCount(); i++) { if (i < 5) { assertThat(result.longValue(i)).isEqualTo(inputProperties.longValue(i)); @@ -165,7 +165,7 @@ void minComponentSizeWithSparseProperties() { var input = inputBuilder.build(); // we mimic the sparseness here through size > valueStored - LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, 3, input::get); + LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, input::get); var config = ConfigWithComponentSize.of(CypherMapWrapper.empty().withNumber("minComponentSize", 2L)); @@ -191,7 +191,7 @@ void consecutiveIdsWithSparseProperties() { var input = inputBuilder.build(); // we mimic the sparseness here through size > valueStored - LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, 3, input::get); + LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, input::get); var config = ConfigWithComponentSize.of(CypherMapWrapper.create(Map.of("consecutiveIds", true))); @@ -221,12 +221,7 @@ private TestNodePropertyValues( } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } @@ -238,26 +233,18 @@ public long longValue(long nodeId) { private static final class TestSparseNodePropertyValues implements LongNodePropertyValues { private final long size; - private final long valuesStored; private final LongToLongFunction transformer; private TestSparseNodePropertyValues( long size, - long valuesStored, LongToLongFunction transformer ) { this.size = size; - this.valuesStored = valuesStored; this.transformer = transformer; } @Override - public long valuesStored() { - return valuesStored; - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java index 93d71f0bedf..066ccec4f13 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/fastrp/FastRPCompanion.java @@ -40,12 +40,7 @@ public float[] floatArrayValue(long nodeId) { } @Override - public long valuesStored() { - return embeddings.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } }; diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java index 7a77d6e283f..99e63730308 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/graphsage/GraphSageCompanion.java @@ -46,12 +46,7 @@ public static DoubleArrayNodePropertyValues getN return new DoubleArrayNodePropertyValues() { @Override - public long valuesStored() { - return embeddings.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return size; } diff --git a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java index e30b5d5f8d7..42c21637171 100644 --- a/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java +++ b/proc/embeddings/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNProcCompanion.java @@ -40,12 +40,7 @@ public double[] doubleArrayValue(long nodeId) { } @Override - public long valuesStored() { - return embeddings.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return nodeCount; } }; diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java index c40859b0cb8..e741daf872d 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineMutateProc.java @@ -97,12 +97,7 @@ protected List nodePropertyList(ComputationResult { var properties = new DoubleArrayNodePropertyValues() { @Override - public long valuesStored() { - return probabilityProperties.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return computationResult.graph().nodeCount(); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java index cd6ef01e29b..c7c4c5e0af4 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/classification/predict/NodeClassificationPipelineWriteProc.java @@ -98,12 +98,7 @@ protected List nodePropertyList(ComputationResult { var properties = new DoubleArrayNodePropertyValues() { @Override - public long valuesStored() { - return probabilityProperties.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { return computationResult.graph().nodeCount(); } diff --git a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java index 2f5803b8d46..9860863fead 100644 --- a/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java +++ b/proc/machine-learning/src/main/java/org/neo4j/gds/ml/pipeline/node/regression/predict/NodeRegressionPipelineMutateProc.java @@ -71,12 +71,7 @@ protected NodePropertyValues nodeProperties(ComputationResult longArrays) {this.longArrays = longArrays;} @Override - public long valuesStored() { - return longArrays.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { // Its backed by a dense array return longArrays.size(); } @@ -186,12 +181,7 @@ static class HugeObjectArrayDoubleArrayPropertyValues implements DoubleArrayNode HugeObjectArrayDoubleArrayPropertyValues(HugeObjectArray doubleArrays) {this.doubleArrays = doubleArrays;} @Override - public long valuesStored() { - return doubleArrays.size(); - } - - @Override - public long maxIndex() { + public long nodeCount() { // its backed by dense array return doubleArrays.size(); } diff --git a/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java b/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java index 8f3dc745863..5221a315a06 100644 --- a/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java +++ b/proc/test/src/main/java/org/neo4j/gds/test/TestMutateProc.java @@ -70,12 +70,7 @@ public long longValue(long nodeId) { } @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return 0; } }; diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java index 1cb790df6aa..213528a36cb 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleArrayTestPropertyValues.java @@ -28,12 +28,7 @@ public final class DoubleArrayTestPropertyValues implements DoubleArrayNodePrope public DoubleArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return -1; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java index 05430bb253c..fae77987bb1 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/DoubleTestPropertyValues.java @@ -29,12 +29,7 @@ public final class DoubleTestPropertyValues implements DoubleNodePropertyValues public DoubleTestPropertyValues(LongToDoubleFunction transformer) {this.transformer = transformer;} @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return 0; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java index de34291ec3b..b1fc69ad19d 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/FloatArrayTestPropertyValues.java @@ -28,12 +28,7 @@ public final class FloatArrayTestPropertyValues implements FloatArrayNodePropert public FloatArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return -1; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java index 2bb4c80047a..ead19184422 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongArrayTestPropertyValues.java @@ -28,12 +28,7 @@ public final class LongArrayTestPropertyValues implements LongArrayNodePropertyV public LongArrayTestPropertyValues(LongToObjectFunction transformer) {this.transformer = transformer;} @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return -1; } diff --git a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java index 6e3891aecea..d6e7964781e 100644 --- a/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java +++ b/test-utils/src/main/java/org/neo4j/gds/nodeproperties/LongTestPropertyValues.java @@ -28,12 +28,7 @@ public final class LongTestPropertyValues implements LongNodePropertyValues { public LongTestPropertyValues(LongToLongFunction transformer) {this.transformer = transformer;} @Override - public long valuesStored() { - return 0; - } - - @Override - public long maxIndex() { + public long nodeCount() { return -1; } From 09a575e559b95c9f918777ceac438164ac1a24fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 6 Mar 2023 16:35:39 +0100 Subject: [PATCH 260/400] Fix nodeCount for NodePropertiesFromStore --- .../nodeproperties/NodePropertiesFromStoreBuilder.java | 6 +----- .../org/neo4j/gds/core/loading/CypherFactoryTest.java | 4 ++-- .../core/loading/NodePropertiesFromStoreBuilderTest.java | 4 ++-- .../org/neo4j/gds/similarity/knn/KnnStreamProcTest.java | 9 ++++----- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java index e9cf0ce61de..df891258ee3 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/NodePropertiesFromStoreBuilder.java @@ -31,7 +31,6 @@ import org.neo4j.values.storable.Values; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.atomic.LongAdder; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; import static org.neo4j.values.storable.Values.NO_VALUE; @@ -63,7 +62,6 @@ public static NodePropertiesFromStoreBuilder of( private final DefaultValue defaultValue; private final int concurrency; private final AtomicReference innerBuilder; - private final LongAdder size; private NodePropertiesFromStoreBuilder( DefaultValue defaultValue, @@ -72,7 +70,6 @@ private NodePropertiesFromStoreBuilder( this.defaultValue = defaultValue; this.concurrency = concurrency; this.innerBuilder = new AtomicReference<>(); - this.size = new LongAdder(); } public void set(long neoNodeId, Value value) { @@ -81,7 +78,6 @@ public void set(long neoNodeId, Value value) { initializeWithType(value); } innerBuilder.get().setValue(neoNodeId, value); - size.increment(); } } @@ -94,7 +90,7 @@ public NodePropertyValues build(IdMap idMap) { } } - return innerBuilder.get().build(this.size.sum(), idMap, idMap.highestOriginalId()); + return innerBuilder.get().build(idMap.nodeCount(), idMap, idMap.highestOriginalId()); } // This is synchronized as we want to prevent the creation of multiple InnerNodePropertiesBuilders of which only once survives. diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java index 6ec6459bfd7..8ecc69f74ea 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/CypherFactoryTest.java @@ -532,8 +532,8 @@ private static Stream memoryEstimationVariants() { "Node properties", "MATCH (n) RETURN id(n) as id, n.id as idProp", "MATCH (n)-[r]->(m) RETURN id(n) AS source, id(m) AS target", - 1300728, - 1300728 + 1300720, + 1300720 ), Arguments.of( diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java index 7d3e02286cb..3b0b98116a3 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NodePropertiesFromStoreBuilderTest.java @@ -61,7 +61,7 @@ void testEmptyDoubleProperties() { 1 ).build(idMap(nodeCount)); - assertEquals(0L, properties.nodeCount()); + assertEquals(nodeCount, properties.nodeCount()); assertEquals(OptionalDouble.empty(), properties.getMaxDoublePropertyValue()); assertEquals(42.0, properties.doubleValue(0)); } @@ -74,7 +74,7 @@ void testEmptyLongProperties() { 1 ).build(idMap(nodeCount)); - assertEquals(0L, properties.nodeCount()); + assertEquals(nodeCount, properties.nodeCount()); assertEquals(OptionalLong.empty(), properties.getMaxLongPropertyValue()); assertEquals(42, properties.longValue(0)); } diff --git a/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java b/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java index a8c4ccdda11..096cee6f177 100644 --- a/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java +++ b/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.similarity.knn; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.AlgoBaseProc; import org.neo4j.gds.GdsCypher; @@ -30,6 +31,8 @@ import java.util.List; import java.util.Map; +import static org.assertj.core.api.Assertions.*; + class KnnStreamProcTest extends KnnProcTest { private static final Collection EXPECTED = new HashSet<>(); @@ -100,7 +103,6 @@ void computeOverSparseNodeProperties() { String nodeCreateQuery = "CREATE " + " (alice:Person {grades: [24, 4]})" + - " ,(carol:Person)" + " ,(eve:Person)" + " ,(bob:Foo {grades: [24, 4, 42]})"; @@ -117,9 +119,6 @@ void computeOverSparseNodeProperties() { .addParameter("nodeProperties", List.of("grades")) .yields("node1", "node2", "similarity"); - assertCypherResult(algoQuery, List.of( - Map.of("node1", 6L, "node2", 7L, "similarity", 1.0), - Map.of("node1", 7L, "node2", 6L, "similarity", 1.0) - )); + runQueryWithResultConsumer(algoQuery, result -> assertThat(result.stream().count()).isEqualTo(2)); } } From e355a7933f33ef20c80d675d3bd6fb0a2fe59e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 7 Mar 2023 10:24:00 +0100 Subject: [PATCH 261/400] Use capacity for default value computation As the size is now the nodeCount and not the number of values set. --- .../core/loading/nodeproperties/LongNodePropertiesBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java index c33877761a3..e743d718d13 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/LongNodePropertiesBuilder.java @@ -128,7 +128,7 @@ public NodePropertyValues build(long size, PartialIdMap idMap, long highestOrigi var propertyValues = propertiesByMappedIdsBuilder.build(); - var maybeMaxValue = size > 0 + var maybeMaxValue = propertyValues.capacity() > 0 ? OptionalLong.of((long) MAX_VALUE.getVolatile(LongNodePropertiesBuilder.this)) : OptionalLong.empty(); From b59cb1c4118e5eaad0cc1f13516c2eda2636254a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 7 Mar 2023 15:06:16 +0100 Subject: [PATCH 262/400] Fix rebase --- .../neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java | 1 - .../gds/nodeproperties/ConsecutiveLongNodePropertyValues.java | 2 +- .../src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java b/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java index f2f1af4e86d..656c25f8541 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/knn/GenerateRandomNeighborsTest.java @@ -23,7 +23,6 @@ import net.jqwik.api.From; import net.jqwik.api.Property; import org.eclipse.collections.api.tuple.primitive.IntIntPair; -import org.neo4j.gds.api.properties.nodes.LongNodePropertyValues; import org.neo4j.gds.core.huge.DirectIdMap; import org.neo4j.gds.core.utils.IdentityPropertyValues; import org.neo4j.gds.core.utils.paged.HugeObjectArray; diff --git a/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java b/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java index 1033d8dbc1c..8575fb175c1 100644 --- a/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java +++ b/proc/common/src/main/java/org/neo4j/gds/nodeproperties/ConsecutiveLongNodePropertyValues.java @@ -49,7 +49,7 @@ public ConsecutiveLongNodePropertyValues(LongNodePropertyValues inputProperties) var communityId = setIdToConsecutiveId.getOrDefault(setId, -1); if (communityId == -1) { //if this is null, it means this community should not be written - if (longNodeProperties.value(nodeId) != null) { + if (inputProperties.value(nodeId) != null) { setIdToConsecutiveId.addTo(setId, ++nextConsecutiveId); communityId = nextConsecutiveId; } else { diff --git a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java index ca6633ff3dc..7b1f6e87270 100644 --- a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java @@ -142,7 +142,7 @@ void shouldWorkWithMinComponentAndConsecutive() { ); - for (long i = 0L; i < result.size(); i++) { + for (long i = 0L; i < result.nodeCount(); i++) { int ii = (int) i; if (returnedValues[ii] != null) { assertThat(result.value(i).asObject()).isEqualTo(returnedValues[ii]); From a28b5062d6e8b95b7372bee0dbd584065b7684e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 7 Mar 2023 15:12:17 +0100 Subject: [PATCH 263/400] Use capacity for default value computation for doubles --- .../loading/nodeproperties/DoubleNodePropertiesBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java index f955721dfb1..e5b45a1e484 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/nodeproperties/DoubleNodePropertiesBuilder.java @@ -120,7 +120,7 @@ public DoubleNodePropertyValues build(long size, PartialIdMap idMap, long highes var propertyValues = propertiesByMappedIdsBuilder.build(); - var maybeMaxValue = size > 0 + var maybeMaxValue = propertyValues.capacity() > 0 ? OptionalDouble.of((double) MAX_VALUE.getVolatile(DoubleNodePropertiesBuilder.this)) : OptionalDouble.empty(); From b381bd7492b0458975583c7979aa2666b3583931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 7 Mar 2023 16:14:12 +0100 Subject: [PATCH 264/400] Fix rebase --- .../test/java/org/neo4j/gds/CommunityProcCompanionTest.java | 4 ++-- .../java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java index 7b1f6e87270..1f9811e0bba 100644 --- a/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java +++ b/proc/community/src/test/java/org/neo4j/gds/CommunityProcCompanionTest.java @@ -167,7 +167,7 @@ void minComponentSizeWithSparseProperties() { // we mimic the sparseness here through size > valueStored LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, input::get); - var config = ConfigWithComponentSize.of(CypherMapWrapper.empty().withNumber("minComponentSize", 2L)); + var config = CommunityProcCompanionConfig.of(CypherMapWrapper.empty().withNumber("minCommunitySize", 2L)); var filteredProperties = CommunityProcCompanion.nodeProperties( config, @@ -193,7 +193,7 @@ void consecutiveIdsWithSparseProperties() { // we mimic the sparseness here through size > valueStored LongNodePropertyValues sparseProperties = new TestSparseNodePropertyValues(4, input::get); - var config = ConfigWithComponentSize.of(CypherMapWrapper.create(Map.of("consecutiveIds", true))); + var config = CommunityProcCompanionConfig.of(CypherMapWrapper.create(Map.of("consecutiveIds", true))); var filteredProperties = CommunityProcCompanion.nodeProperties( config, diff --git a/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java b/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java index 096cee6f177..d21b7d202d0 100644 --- a/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java +++ b/proc/similarity/src/test/java/org/neo4j/gds/similarity/knn/KnnStreamProcTest.java @@ -19,7 +19,6 @@ */ package org.neo4j.gds.similarity.knn; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.AlgoBaseProc; import org.neo4j.gds.GdsCypher; @@ -31,7 +30,7 @@ import java.util.List; import java.util.Map; -import static org.assertj.core.api.Assertions.*; +import static org.assertj.core.api.Assertions.assertThat; class KnnStreamProcTest extends KnnProcTest { From e2439af411e8cc075e864f0d69772bac0f41ea81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Mon, 6 Mar 2023 11:47:44 +0100 Subject: [PATCH 265/400] Add fake impl of SEFactory to 5.x layers Co-authored-by: Paul Horn --- .../5.1/storage-engine-adapter/build.gradle | 1 + .../_51/InMemoryStorageEngineFactory.java | 272 ++++++++++++++++++ .../5.2/storage-engine-adapter/build.gradle | 1 + .../_52/InMemoryStorageEngineFactory.java | 272 ++++++++++++++++++ .../5.3/storage-engine-adapter/build.gradle | 1 + .../_53/InMemoryStorageEngineFactory.java | 272 ++++++++++++++++++ .../5.4/storage-engine-adapter/build.gradle | 1 + .../_54/InMemoryStorageEngineFactory.java | 272 ++++++++++++++++++ .../5.5/storage-engine-adapter/build.gradle | 1 + .../_55/InMemoryStorageEngineFactory.java | 272 ++++++++++++++++++ 10 files changed, 1365 insertions(+) create mode 100644 compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java diff --git a/compatibility/5.1/storage-engine-adapter/build.gradle b/compatibility/5.1/storage-engine-adapter/build.gradle index ac0d4da3b60..d9dcbe96a38 100644 --- a/compatibility/5.1/storage-engine-adapter/build.gradle +++ b/compatibility/5.1/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..c527849ab92 --- /dev/null +++ b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._51; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + public InMemoryStorageEngineFactory() { + throw new UnsupportedOperationException("This should never be instantiated"); + } + + @Override + public String name() { + return null; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return null; + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + return null; + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + return null; + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + return null; + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + return null; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + return null; + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + return Optional.empty(); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + return null; + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + return null; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return null; + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + return null; + } +} diff --git a/compatibility/5.2/storage-engine-adapter/build.gradle b/compatibility/5.2/storage-engine-adapter/build.gradle index 8af17934fc0..0bc2a320eba 100644 --- a/compatibility/5.2/storage-engine-adapter/build.gradle +++ b/compatibility/5.2/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..61c24b4ead0 --- /dev/null +++ b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._52; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + public InMemoryStorageEngineFactory() { + throw new UnsupportedOperationException("This should never be instantiated"); + } + + @Override + public String name() { + return null; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return null; + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + return null; + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + return null; + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + return null; + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + return null; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + return null; + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + return Optional.empty(); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + return null; + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + return null; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return null; + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + return null; + } +} diff --git a/compatibility/5.3/storage-engine-adapter/build.gradle b/compatibility/5.3/storage-engine-adapter/build.gradle index 805b935c705..6db56beaaba 100644 --- a/compatibility/5.3/storage-engine-adapter/build.gradle +++ b/compatibility/5.3/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..4fbd90e8cf6 --- /dev/null +++ b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._53; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + public InMemoryStorageEngineFactory() { + throw new UnsupportedOperationException("This should never be instantiated"); + } + + @Override + public String name() { + return null; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return null; + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + return null; + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + return null; + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + return null; + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + return null; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + return null; + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + return Optional.empty(); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + return null; + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + return null; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return null; + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + return null; + } +} diff --git a/compatibility/5.4/storage-engine-adapter/build.gradle b/compatibility/5.4/storage-engine-adapter/build.gradle index 0feaddfc02f..151d3362ee9 100644 --- a/compatibility/5.4/storage-engine-adapter/build.gradle +++ b/compatibility/5.4/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..1432f4f3ad1 --- /dev/null +++ b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._54; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + public InMemoryStorageEngineFactory() { + throw new UnsupportedOperationException("This should never be instantiated"); + } + + @Override + public String name() { + return null; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return null; + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + return null; + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + return null; + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + return null; + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + return null; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + return null; + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + return Optional.empty(); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + return null; + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + return null; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return null; + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + return null; + } +} diff --git a/compatibility/5.5/storage-engine-adapter/build.gradle b/compatibility/5.5/storage-engine-adapter/build.gradle index 57c683e1fd5..330a0408d2c 100644 --- a/compatibility/5.5/storage-engine-adapter/build.gradle +++ b/compatibility/5.5/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..dfbfa67e14a --- /dev/null +++ b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._55; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + public InMemoryStorageEngineFactory() { + throw new UnsupportedOperationException("This should never be instantiated"); + } + + @Override + public String name() { + return null; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return null; + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + return null; + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + return null; + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + return null; + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + return null; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + return null; + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return null; + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + return Optional.empty(); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + return null; + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + return null; + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + return null; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return null; + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + return null; + } +} From 1b5e8883167b60085cd4c99a6b4ab5fedb6b607e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Mon, 6 Mar 2023 17:04:30 +0100 Subject: [PATCH 266/400] Throw on each method instead of constructor --- .../_51/InMemoryStorageEngineFactory.java | 46 +++++++++---------- .../_52/InMemoryStorageEngineFactory.java | 46 +++++++++---------- .../_53/InMemoryStorageEngineFactory.java | 46 +++++++++---------- .../_54/InMemoryStorageEngineFactory.java | 46 +++++++++---------- .../_55/InMemoryStorageEngineFactory.java | 46 +++++++++---------- 5 files changed, 105 insertions(+), 125 deletions(-) diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java index c527849ab92..8b8b61fbbe1 100644 --- a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java @@ -65,13 +65,9 @@ @ServiceProvider public class InMemoryStorageEngineFactory implements StorageEngineFactory { - public InMemoryStorageEngineFactory() { - throw new UnsupportedOperationException("This should never be instantiated"); - } - @Override public String name() { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -83,22 +79,22 @@ public StoreVersionCheck versionCheck( LogService logService, PageCacheTracer pageCacheTracer ) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(String storeVersion) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(StoreId storeId) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override public RollingUpgradeCompatibility rollingUpgradeCompatibility() { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -111,7 +107,7 @@ public List migrationParticipants( PageCacheTracer cacheTracer, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -136,18 +132,18 @@ public StorageEngine instantiate( DatabaseReadOnlyChecker readOnlyChecker, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws IOException { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - return false; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -157,7 +153,7 @@ public TransactionIdStore readOnlyTransactionIdStore( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -166,7 +162,7 @@ public LogVersionRepository readOnlyLogVersionRepository( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -178,7 +174,7 @@ public MetadataProvider transactionMetaDataStore( PageCacheTracer cacheTracer, DatabaseReadOnlyChecker readOnlyChecker ) throws IOException { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -188,7 +184,7 @@ public StoreId storeId( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -201,7 +197,7 @@ public void setStoreId( long upgradeTxChecksum, long upgradeTxCommitTimestamp ) throws IOException { - + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -212,7 +208,7 @@ public void setExternalStoreUUID( CursorContext cursorContext, UUID externalStoreId ) throws IOException { - + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -222,7 +218,7 @@ public Optional databaseIdUuid( PageCache pageCache, CursorContext cursorContext ) { - return Optional.empty(); + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -237,7 +233,7 @@ public SchemaRuleMigrationAccess schemaRuleMigrationAccess( CursorContext cursorContext, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -248,7 +244,7 @@ public List loadSchemaRules( DatabaseLayout databaseLayout, CursorContext cursorContext ) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override @@ -257,16 +253,16 @@ public StorageFilesState checkStoreFileState( DatabaseLayout databaseLayout, PageCache pageCache ) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override public CommandReaderFactory commandReaderFactory() { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } @Override public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { - return null; + throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); } } diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java index 61c24b4ead0..ae785449310 100644 --- a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java @@ -65,13 +65,9 @@ @ServiceProvider public class InMemoryStorageEngineFactory implements StorageEngineFactory { - public InMemoryStorageEngineFactory() { - throw new UnsupportedOperationException("This should never be instantiated"); - } - @Override public String name() { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -83,22 +79,22 @@ public StoreVersionCheck versionCheck( LogService logService, PageCacheTracer pageCacheTracer ) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(String storeVersion) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(StoreId storeId) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override public RollingUpgradeCompatibility rollingUpgradeCompatibility() { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -111,7 +107,7 @@ public List migrationParticipants( PageCacheTracer cacheTracer, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -136,18 +132,18 @@ public StorageEngine instantiate( DatabaseReadOnlyChecker readOnlyChecker, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws IOException { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - return false; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -157,7 +153,7 @@ public TransactionIdStore readOnlyTransactionIdStore( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -166,7 +162,7 @@ public LogVersionRepository readOnlyLogVersionRepository( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -178,7 +174,7 @@ public MetadataProvider transactionMetaDataStore( PageCacheTracer cacheTracer, DatabaseReadOnlyChecker readOnlyChecker ) throws IOException { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -188,7 +184,7 @@ public StoreId storeId( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -201,7 +197,7 @@ public void setStoreId( long upgradeTxChecksum, long upgradeTxCommitTimestamp ) throws IOException { - + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -212,7 +208,7 @@ public void setExternalStoreUUID( CursorContext cursorContext, UUID externalStoreId ) throws IOException { - + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -222,7 +218,7 @@ public Optional databaseIdUuid( PageCache pageCache, CursorContext cursorContext ) { - return Optional.empty(); + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -237,7 +233,7 @@ public SchemaRuleMigrationAccess schemaRuleMigrationAccess( CursorContext cursorContext, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -248,7 +244,7 @@ public List loadSchemaRules( DatabaseLayout databaseLayout, CursorContext cursorContext ) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override @@ -257,16 +253,16 @@ public StorageFilesState checkStoreFileState( DatabaseLayout databaseLayout, PageCache pageCache ) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override public CommandReaderFactory commandReaderFactory() { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } @Override public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { - return null; + throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); } } diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java index 4fbd90e8cf6..786aa73fffa 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -65,13 +65,9 @@ @ServiceProvider public class InMemoryStorageEngineFactory implements StorageEngineFactory { - public InMemoryStorageEngineFactory() { - throw new UnsupportedOperationException("This should never be instantiated"); - } - @Override public String name() { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -83,22 +79,22 @@ public StoreVersionCheck versionCheck( LogService logService, PageCacheTracer pageCacheTracer ) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(String storeVersion) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(StoreId storeId) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override public RollingUpgradeCompatibility rollingUpgradeCompatibility() { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -111,7 +107,7 @@ public List migrationParticipants( PageCacheTracer cacheTracer, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -136,18 +132,18 @@ public StorageEngine instantiate( DatabaseReadOnlyChecker readOnlyChecker, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws IOException { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - return false; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -157,7 +153,7 @@ public TransactionIdStore readOnlyTransactionIdStore( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -166,7 +162,7 @@ public LogVersionRepository readOnlyLogVersionRepository( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -178,7 +174,7 @@ public MetadataProvider transactionMetaDataStore( PageCacheTracer cacheTracer, DatabaseReadOnlyChecker readOnlyChecker ) throws IOException { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -188,7 +184,7 @@ public StoreId storeId( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -201,7 +197,7 @@ public void setStoreId( long upgradeTxChecksum, long upgradeTxCommitTimestamp ) throws IOException { - + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -212,7 +208,7 @@ public void setExternalStoreUUID( CursorContext cursorContext, UUID externalStoreId ) throws IOException { - + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -222,7 +218,7 @@ public Optional databaseIdUuid( PageCache pageCache, CursorContext cursorContext ) { - return Optional.empty(); + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -237,7 +233,7 @@ public SchemaRuleMigrationAccess schemaRuleMigrationAccess( CursorContext cursorContext, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -248,7 +244,7 @@ public List loadSchemaRules( DatabaseLayout databaseLayout, CursorContext cursorContext ) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override @@ -257,16 +253,16 @@ public StorageFilesState checkStoreFileState( DatabaseLayout databaseLayout, PageCache pageCache ) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override public CommandReaderFactory commandReaderFactory() { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } @Override public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { - return null; + throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); } } diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java index 1432f4f3ad1..a60a0bed335 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -65,13 +65,9 @@ @ServiceProvider public class InMemoryStorageEngineFactory implements StorageEngineFactory { - public InMemoryStorageEngineFactory() { - throw new UnsupportedOperationException("This should never be instantiated"); - } - @Override public String name() { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -83,22 +79,22 @@ public StoreVersionCheck versionCheck( LogService logService, PageCacheTracer pageCacheTracer ) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(String storeVersion) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(StoreId storeId) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override public RollingUpgradeCompatibility rollingUpgradeCompatibility() { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -111,7 +107,7 @@ public List migrationParticipants( PageCacheTracer cacheTracer, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -136,18 +132,18 @@ public StorageEngine instantiate( DatabaseReadOnlyChecker readOnlyChecker, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws IOException { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - return false; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -157,7 +153,7 @@ public TransactionIdStore readOnlyTransactionIdStore( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -166,7 +162,7 @@ public LogVersionRepository readOnlyLogVersionRepository( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -178,7 +174,7 @@ public MetadataProvider transactionMetaDataStore( PageCacheTracer cacheTracer, DatabaseReadOnlyChecker readOnlyChecker ) throws IOException { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -188,7 +184,7 @@ public StoreId storeId( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -201,7 +197,7 @@ public void setStoreId( long upgradeTxChecksum, long upgradeTxCommitTimestamp ) throws IOException { - + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -212,7 +208,7 @@ public void setExternalStoreUUID( CursorContext cursorContext, UUID externalStoreId ) throws IOException { - + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -222,7 +218,7 @@ public Optional databaseIdUuid( PageCache pageCache, CursorContext cursorContext ) { - return Optional.empty(); + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -237,7 +233,7 @@ public SchemaRuleMigrationAccess schemaRuleMigrationAccess( CursorContext cursorContext, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -248,7 +244,7 @@ public List loadSchemaRules( DatabaseLayout databaseLayout, CursorContext cursorContext ) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override @@ -257,16 +253,16 @@ public StorageFilesState checkStoreFileState( DatabaseLayout databaseLayout, PageCache pageCache ) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override public CommandReaderFactory commandReaderFactory() { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } @Override public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { - return null; + throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); } } diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java index dfbfa67e14a..29af6c11f01 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -65,13 +65,9 @@ @ServiceProvider public class InMemoryStorageEngineFactory implements StorageEngineFactory { - public InMemoryStorageEngineFactory() { - throw new UnsupportedOperationException("This should never be instantiated"); - } - @Override public String name() { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -83,22 +79,22 @@ public StoreVersionCheck versionCheck( LogService logService, PageCacheTracer pageCacheTracer ) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(String storeVersion) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override public StoreVersion versionInformation(StoreId storeId) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override public RollingUpgradeCompatibility rollingUpgradeCompatibility() { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -111,7 +107,7 @@ public List migrationParticipants( PageCacheTracer cacheTracer, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -136,18 +132,18 @@ public StorageEngine instantiate( DatabaseReadOnlyChecker readOnlyChecker, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws IOException { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - return false; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -157,7 +153,7 @@ public TransactionIdStore readOnlyTransactionIdStore( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -166,7 +162,7 @@ public LogVersionRepository readOnlyLogVersionRepository( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -178,7 +174,7 @@ public MetadataProvider transactionMetaDataStore( PageCacheTracer cacheTracer, DatabaseReadOnlyChecker readOnlyChecker ) throws IOException { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -188,7 +184,7 @@ public StoreId storeId( PageCache pageCache, CursorContext cursorContext ) throws IOException { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -201,7 +197,7 @@ public void setStoreId( long upgradeTxChecksum, long upgradeTxCommitTimestamp ) throws IOException { - + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -212,7 +208,7 @@ public void setExternalStoreUUID( CursorContext cursorContext, UUID externalStoreId ) throws IOException { - + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -222,7 +218,7 @@ public Optional databaseIdUuid( PageCache pageCache, CursorContext cursorContext ) { - return Optional.empty(); + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -237,7 +233,7 @@ public SchemaRuleMigrationAccess schemaRuleMigrationAccess( CursorContext cursorContext, MemoryTracker memoryTracker ) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -248,7 +244,7 @@ public List loadSchemaRules( DatabaseLayout databaseLayout, CursorContext cursorContext ) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override @@ -257,16 +253,16 @@ public StorageFilesState checkStoreFileState( DatabaseLayout databaseLayout, PageCache pageCache ) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override public CommandReaderFactory commandReaderFactory() { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } @Override public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { - return null; + throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); } } From 01099ffd074331fed79bd1d9bd751c11d07526ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Tue, 7 Mar 2023 10:31:05 +0100 Subject: [PATCH 267/400] Implement StorageEngine name with unsupported --- .../org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java index 8b8b61fbbe1..4fd30ed710a 100644 --- a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + return "unsupported"; } @Override diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java index ae785449310..7256a4276b9 100644 --- a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + return "unsupported"; } @Override diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java index 786aa73fffa..5972a4ab0a8 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + return "unsupported"; } @Override diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java index a60a0bed335..a7ad9aae965 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + return "unsupported"; } @Override diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java index 29af6c11f01..2f672d3ea2e 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + return "unsupported"; } @Override From 84edf97e7cc95e692f22f5885d9a2f7fb3a4501e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Tue, 7 Mar 2023 14:53:25 +0100 Subject: [PATCH 268/400] Implement storage exists method --- .../org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java index 4fd30ed710a..571dbe83d71 100644 --- a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java @@ -143,7 +143,7 @@ public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLay @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - throw new UnsupportedOperationException("5.1 storage engine requires JDK17"); + return false; } @Override diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java index 7256a4276b9..fc98e5bd6a0 100644 --- a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java @@ -143,7 +143,7 @@ public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLay @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - throw new UnsupportedOperationException("5.2 storage engine requires JDK17"); + return false; } @Override diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java index 5972a4ab0a8..d47ca364a9b 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -143,7 +143,7 @@ public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLay @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - throw new UnsupportedOperationException("5.3 storage engine requires JDK17"); + return false; } @Override diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java index a7ad9aae965..4e257bea6e0 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -143,7 +143,7 @@ public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLay @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - throw new UnsupportedOperationException("5.4 storage engine requires JDK17"); + return false; } @Override diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java index 2f672d3ea2e..b1344c3dc96 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -143,7 +143,7 @@ public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLay @Override public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { - throw new UnsupportedOperationException("5.5 storage engine requires JDK17"); + return false; } @Override From 223b880ea5f7c16f9be7bc31c55a4397a6040cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Tue, 7 Mar 2023 16:34:54 +0100 Subject: [PATCH 269/400] Satisfy unique storage engine name constraint --- .../org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java | 2 +- .../org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java index 571dbe83d71..c416233af83 100644 --- a/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java +++ b/compatibility/5.1/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_51/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - return "unsupported"; + return "unsupported51"; } @Override diff --git a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java index fc98e5bd6a0..499d70a28a4 100644 --- a/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java +++ b/compatibility/5.2/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_52/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - return "unsupported"; + return "unsupported52"; } @Override diff --git a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java index d47ca364a9b..eada03549ce 100644 --- a/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java +++ b/compatibility/5.3/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_53/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - return "unsupported"; + return "unsupported53"; } @Override diff --git a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java index 4e257bea6e0..23fb737630d 100644 --- a/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java +++ b/compatibility/5.4/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_54/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - return "unsupported"; + return "unsupported54"; } @Override diff --git a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java index b1344c3dc96..177fe8e896b 100644 --- a/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java +++ b/compatibility/5.5/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_55/InMemoryStorageEngineFactory.java @@ -67,7 +67,7 @@ public class InMemoryStorageEngineFactory implements StorageEngineFactory { @Override public String name() { - return "unsupported"; + return "unsupported55"; } @Override From fe6eafeb01bdab586275cf53169d1a8002faa2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 8 Mar 2023 11:38:52 +0100 Subject: [PATCH 270/400] Remove unused code --- .../loading/CypherRelationshipLoader.java | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java index f372a453a15..fd081f80ea5 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherRelationshipLoader.java @@ -19,15 +19,12 @@ */ package org.neo4j.gds.core.loading; -import org.apache.commons.lang3.mutable.MutableInt; import org.eclipse.collections.impl.map.mutable.primitive.ObjectDoubleHashMap; -import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap; import org.immutables.value.Value; import org.neo4j.gds.ImmutablePropertyMappings; import org.neo4j.gds.Orientation; import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.PropertyMappings; -import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphLoaderContext; @@ -52,14 +49,9 @@ class CypherRelationshipLoader extends CypherRecordLoader propertyKeyIdsByName; private ObjectDoubleHashMap propertyDefaultValueByName; private boolean initializedFromResult; private List propertyConfigs; - private RelationshipProjection.Builder projectionBuilder; CypherRelationshipLoader( String relationshipQuery, @@ -75,15 +67,7 @@ class CypherRelationshipLoader extends CypherRecordLoader(numberOfMappings); - propertyMappings - .stream() - .forEach(mapping -> propertyKeyIdsByName.put(mapping.neoPropertyKey(), propertyKeyId.getAndIncrement())); - - propertyDefaultValueByName = new ObjectDoubleHashMap<>(numberOfMappings); + propertyDefaultValueByName = new ObjectDoubleHashMap<>(propertyMappings.numberOfMappings()); propertyMappings .stream() .forEach(mapping -> propertyDefaultValueByName.put( @@ -99,11 +83,6 @@ private void initFromPropertyMappings(PropertyMappings propertyMappings) { mapping.defaultValue() )) .collect(Collectors.toList()); - - projectionBuilder = RelationshipProjection - .builder() - .orientation(Orientation.NATURAL) - .properties(propertyMappings); } @Override From ba29b9785c4a36c5909965f905bc5fe6cb7d8718 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 8 Mar 2023 15:22:55 +0100 Subject: [PATCH 271/400] Use estimated nodes and rels in progress task of Cypher projection Before we used a estimation based on the native projection logic. This lead to underestimating the counts especially for path queries. --- .../neo4j/gds/api/CSRGraphStoreFactory.java | 5 +- .../org/neo4j/gds/api/GraphStoreFactory.java | 5 - .../config/GraphProjectFromCypherConfig.java | 4 +- .../gds/core/GraphDimensionsCypherReader.java | 63 ------ .../neo4j/gds/core/loading/CypherFactory.java | 205 +++++++----------- .../core/loading/CypherQueryEstimator.java | 97 +++++++++ .../neo4j/gds/core/loading/NativeFactory.java | 10 +- .../org/neo4j/gds/core/GraphLoaderTest.java | 14 +- .../java/org/neo4j/gds/gdl/GdlFactory.java | 10 +- 9 files changed, 198 insertions(+), 215 deletions(-) delete mode 100644 core/src/main/java/org/neo4j/gds/core/GraphDimensionsCypherReader.java create mode 100644 core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java diff --git a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java index bc07eeac82a..315d5d91dfc 100644 --- a/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/CSRGraphStoreFactory.java @@ -27,6 +27,7 @@ import org.neo4j.gds.core.loading.GraphStoreBuilder; import org.neo4j.gds.core.loading.Nodes; import org.neo4j.gds.core.loading.RelationshipImportResult; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.mem.MemoryUsage; import java.util.Map; @@ -65,7 +66,9 @@ protected void logLoadingSummary(GraphStore graphStore) { var sizeInBytes = MemoryUsage.sizeOf(graphStore); if (sizeInBytes >= 0) { var memoryUsage = MemoryUsage.humanReadable(sizeInBytes); - progressTracker.logInfo(formatWithLocale("Actual memory usage of the loaded graph: %s", memoryUsage)); + progressTracker().logInfo(formatWithLocale("Actual memory usage of the loaded graph: %s", memoryUsage)); } } + + protected abstract ProgressTracker progressTracker(); } diff --git a/core/src/main/java/org/neo4j/gds/api/GraphStoreFactory.java b/core/src/main/java/org/neo4j/gds/api/GraphStoreFactory.java index b77261b1af3..3ac0a775704 100644 --- a/core/src/main/java/org/neo4j/gds/api/GraphStoreFactory.java +++ b/core/src/main/java/org/neo4j/gds/api/GraphStoreFactory.java @@ -24,7 +24,6 @@ import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.loading.Capabilities; import org.neo4j.gds.core.utils.mem.MemoryEstimation; -import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; /** * The Abstract Factory defines the construction of the graph @@ -46,7 +45,6 @@ public interface Supplier { protected final Capabilities capabilities; protected final GraphLoaderContext loadingContext; protected final GraphDimensions dimensions; - protected final ProgressTracker progressTracker; public GraphStoreFactory( CONFIG graphProjectConfig, @@ -58,7 +56,6 @@ public GraphStoreFactory( this.capabilities = capabilities; this.loadingContext = loadingContext; this.dimensions = dimensions; - this.progressTracker = initProgressTracker(); } public abstract STORE build(); @@ -79,8 +76,6 @@ public CONFIG graphProjectConfig() { return graphProjectConfig; } - protected abstract ProgressTracker initProgressTracker(); - @ValueClass public interface ImportResult { STORE graphStore(); diff --git a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromCypherConfig.java b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromCypherConfig.java index 22142f0b52a..876a8daeb8c 100644 --- a/core/src/main/java/org/neo4j/gds/config/GraphProjectFromCypherConfig.java +++ b/core/src/main/java/org/neo4j/gds/config/GraphProjectFromCypherConfig.java @@ -80,14 +80,14 @@ default GraphStoreFactory.Supplier graphStoreFactory() { return new GraphStoreFactory.Supplier() { @Override public GraphStoreFactory get(GraphLoaderContext loaderContext) { - return new CypherFactory(GraphProjectFromCypherConfig.this, loaderContext); + return CypherFactory.createWithDerivedDimensions(GraphProjectFromCypherConfig.this, loaderContext); } @Override public GraphStoreFactory getWithDimension( GraphLoaderContext loaderContext, GraphDimensions graphDimensions ) { - return new CypherFactory(GraphProjectFromCypherConfig.this, loaderContext, graphDimensions); + return CypherFactory.createWithBaseDimensions(GraphProjectFromCypherConfig.this, loaderContext, graphDimensions); } }; } diff --git a/core/src/main/java/org/neo4j/gds/core/GraphDimensionsCypherReader.java b/core/src/main/java/org/neo4j/gds/core/GraphDimensionsCypherReader.java deleted file mode 100644 index f3127f4da07..00000000000 --- a/core/src/main/java/org/neo4j/gds/core/GraphDimensionsCypherReader.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.core; - -import org.neo4j.gds.NodeLabel; -import org.neo4j.gds.NodeProjections; -import org.neo4j.gds.RelationshipProjections; -import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.config.GraphProjectFromCypherConfig; -import org.neo4j.gds.transaction.TransactionContext; -import org.neo4j.internal.id.IdGeneratorFactory; -import org.neo4j.internal.kernel.api.TokenRead; - -import static org.neo4j.gds.core.GraphDimensions.ANY_LABEL; -import static org.neo4j.gds.core.GraphDimensions.ANY_RELATIONSHIP_TYPE; - -public class GraphDimensionsCypherReader extends GraphDimensionsReader { - - public GraphDimensionsCypherReader( - TransactionContext tx, - GraphProjectFromCypherConfig config, - IdGeneratorFactory idGeneratorFactory - ) { - super(tx, config, idGeneratorFactory); - } - - @Override - protected TokenElementIdentifierMappings getNodeLabelTokens(TokenRead tokenRead) { - return new TokenElementIdentifierMappings<>(ANY_LABEL); - } - - @Override - protected TokenElementIdentifierMappings getRelationshipTypeTokens(TokenRead tokenRead) { - return new TokenElementIdentifierMappings<>(ANY_RELATIONSHIP_TYPE); - } - - @Override - protected NodeProjections getNodeProjections() { - return NodeProjections.all(); - } - - @Override - protected RelationshipProjections getRelationshipProjections() { - return RelationshipProjections.ALL; - } -} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index a91e657fc2d..17a5fb35b94 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -19,74 +19,102 @@ */ package org.neo4j.gds.core.loading; -import org.immutables.value.Value; import org.neo4j.gds.ElementProjection; import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.NodeProjections; -import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.CSRGraphStoreFactory; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphLoaderContext; -import org.neo4j.gds.compat.GraphDatabaseApiProxy; -import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.core.GraphDimensions; -import org.neo4j.gds.core.GraphDimensionsCypherReader; import org.neo4j.gds.core.ImmutableGraphDimensions; import org.neo4j.gds.core.utils.mem.MemoryEstimation; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.TaskProgressTracker; +import org.neo4j.gds.core.utils.progress.tasks.TaskTreeProgressTracker; import org.neo4j.gds.core.utils.progress.tasks.Tasks; import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory; import org.neo4j.gds.transaction.TransactionContext; -import org.neo4j.internal.id.IdGeneratorFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.LongStream; +import javax.annotation.Nullable; -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; +import static org.neo4j.gds.core.loading.CypherQueryEstimator.EstimationResult; import static org.neo4j.internal.kernel.api.security.AccessMode.Static.READ; -import static org.neo4j.kernel.api.StatementConstants.NO_SUCH_PROPERTY_KEY; -public class CypherFactory extends CSRGraphStoreFactory { +public final class CypherFactory extends CSRGraphStoreFactory { private final GraphProjectFromCypherConfig cypherConfig; - private EstimationResult nodeEstimation; - private EstimationResult relationshipEstimation; + private final EstimationResult nodeEstimation; + private final EstimationResult relationshipEstimation; + private final ProgressTracker progressTracker; - public CypherFactory( + public static CypherFactory createWithBaseDimensions(GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext, GraphDimensions graphDimensions) { + return create(graphProjectConfig, loadingContext, graphDimensions); + } + + public static CypherFactory createWithDerivedDimensions(GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext) { + return create(graphProjectConfig, loadingContext, null); + } + + private static CypherFactory create( GraphProjectFromCypherConfig graphProjectConfig, - GraphLoaderContext loadingContext + GraphLoaderContext loadingContext, + @Nullable GraphDimensions dimensions ) { - this( + var estimator = new CypherQueryEstimator(loadingContext.transactionContext().withRestrictedAccess(READ)); + + EstimationResult nodeQueryEstimation = graphProjectConfig.isFictitiousLoading() + ? ImmutableEstimationResult.of(graphProjectConfig.nodeCount(), 0) + : estimator.getNodeEstimation(graphProjectConfig.nodeQuery()); + + EstimationResult relationshipQueryEstimation = graphProjectConfig.isFictitiousLoading() + ? ImmutableEstimationResult.of(graphProjectConfig.relationshipCount(), 0) + : estimator.getRelationshipEstimation(graphProjectConfig.relationshipQuery()); + + var dimBuilder = ImmutableGraphDimensions.builder(); + + if (dimensions != null) { + dimBuilder.from(dimensions); + } + + GraphDimensions dim = ImmutableGraphDimensions.builder() + .highestPossibleNodeCount(nodeQueryEstimation.estimatedRows()) + .nodeCount(nodeQueryEstimation.estimatedRows()) + .relCountUpperBound(relationshipQueryEstimation.estimatedRows()) + .build(); + + return new CypherFactory( graphProjectConfig, loadingContext, - new GraphDimensionsCypherReader( - loadingContext.transactionContext().withRestrictedAccess(READ), - graphProjectConfig, - GraphDatabaseApiProxy.resolveDependency(loadingContext.graphDatabaseService(), IdGeneratorFactory.class) - ).call() + dim, + nodeQueryEstimation, + relationshipQueryEstimation ); } - public CypherFactory( + private CypherFactory( GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext, - GraphDimensions graphDimensions + GraphDimensions graphDimensions, + EstimationResult nodeEstimation, + EstimationResult relationshipEstimation + ) { // TODO: need to pass capabilities from outside? super(graphProjectConfig, ImmutableStaticCapabilities.of(true), loadingContext, graphDimensions); - this.cypherConfig = getCypherConfig(graphProjectConfig).orElseThrow(() -> new IllegalArgumentException( - "Expected GraphProjectConfig to be a cypher config.")); + + this.cypherConfig = graphProjectConfig; + this.nodeEstimation = nodeEstimation; + this.relationshipEstimation = relationshipEstimation; + this.progressTracker = initProgressTracker(); + } + + @Override + protected ProgressTracker progressTracker() { + return progressTracker; } @Override @@ -111,9 +139,9 @@ public MemoryEstimation estimateMemoryUsageAfterLoading() { public GraphDimensions estimationDimensions() { return ImmutableGraphDimensions.builder() .from(dimensions) - .highestPossibleNodeCount(getNodeEstimation().estimatedRows()) - .nodeCount(getNodeEstimation().estimatedRows()) - .relCountUpperBound(getRelationshipEstimation().estimatedRows()) + .highestPossibleNodeCount(Math.max(dimensions.highestPossibleNodeCount(), nodeEstimation.estimatedRows())) + .nodeCount(Math.max(dimensions.nodeCount(), nodeEstimation.estimatedRows())) + .relCountUpperBound(Math.max(dimensions.relCountUpperBound(), relationshipEstimation.estimatedRows())) .build(); } @@ -122,7 +150,7 @@ public CSRGraphStore build() { // Temporarily override the security context to enforce read-only access during load return readOnlyTransaction().apply((tx, ktx) -> { BatchLoadResult nodeCount = new CountingCypherRecordLoader( - nodeQuery(), + cypherConfig.nodeQuery(), CypherRecordLoader.QueryType.NODE, cypherConfig, loadingContext @@ -130,7 +158,7 @@ public CSRGraphStore build() { progressTracker.beginSubTask("Loading"); var nodes = new CypherNodeLoader( - nodeQuery(), + cypherConfig.nodeQuery(), nodeCount.rows(), cypherConfig, loadingContext, @@ -138,7 +166,7 @@ public CSRGraphStore build() { ).load(ktx.internalTransaction()); var relationshipImportResult = new CypherRelationshipLoader( - relationshipQuery(), + cypherConfig.relationshipQuery(), nodes.idMap(), cypherConfig, loadingContext, @@ -158,13 +186,14 @@ public CSRGraphStore build() { }); } - @Override - protected ProgressTracker initProgressTracker() { + private ProgressTracker initProgressTracker() { + var estimatedDimensions = estimationDimensions(); var task = Tasks.task( "Loading", - Tasks.leaf("Nodes"), - Tasks.leaf("Relationships", dimensions.relCountUpperBound()) + Tasks.leaf("Nodes", estimatedDimensions.highestPossibleNodeCount()), + Tasks.leaf("Relationships", estimatedDimensions.relCountUpperBound()) ); + if (graphProjectConfig.logProgress()) { return new TaskProgressTracker( task, @@ -176,7 +205,7 @@ protected ProgressTracker initProgressTracker() { ); } - return new TaskProgressTracker( + return new TaskTreeProgressTracker( task, loadingContext.log(), graphProjectConfig.readConcurrency(), @@ -186,72 +215,15 @@ protected ProgressTracker initProgressTracker() { ); } - private String nodeQuery() { - return getCypherConfig(graphProjectConfig) - .orElseThrow(() -> new IllegalArgumentException("Missing node query")) - .nodeQuery(); - } - - private String relationshipQuery() { - return getCypherConfig(graphProjectConfig) - .orElseThrow(() -> new IllegalArgumentException("Missing relationship query")) - .relationshipQuery(); - } - - private static Optional getCypherConfig(GraphProjectConfig config) { - if (config instanceof GraphProjectFromCypherConfig) { - return Optional.of((GraphProjectFromCypherConfig) config); - } - return Optional.empty(); - } - private TransactionContext readOnlyTransaction() { return loadingContext.transactionContext().withRestrictedAccess(READ); } - private EstimationResult getNodeEstimation() { - if (nodeEstimation == null) { - nodeEstimation = runEstimationQuery( - nodeQuery(), - NodeSubscriber.RESERVED_COLUMNS - ); - } - return nodeEstimation; - } - - private EstimationResult getRelationshipEstimation() { - if (relationshipEstimation == null) { - relationshipEstimation = runEstimationQuery( - relationshipQuery(), - RelationshipSubscriber.RESERVED_COLUMNS - ); - } - return relationshipEstimation; - } - - private EstimationResult runEstimationQuery(String query, Collection reservedColumns) { - return readOnlyTransaction().apply((tx, ktx) -> { - var explainQuery = formatWithLocale("EXPLAIN %s", query); - try (var result = tx.execute(explainQuery)) { - var estimatedRows = (Number) result.getExecutionPlanDescription().getArguments().get("EstimatedRows"); - - var propertyColumns = new ArrayList<>(result.columns()); - propertyColumns.removeAll(reservedColumns); - - return ImmutableEstimationResult.of(estimatedRows.longValue(), propertyColumns.size()); - } - }); - } - private NodeProjections buildEstimateNodeProjections() { - if (cypherConfig.isFictitiousLoading()) { - nodeEstimation = ImmutableEstimationResult.of(cypherConfig.nodeCount(), 0); - } - var nodeProjection = NodeProjection .builder() .label(ElementProjection.PROJECT_ALL) - .addAllProperties(getNodeEstimation().propertyMappings()) + .addAllProperties(nodeEstimation.propertyMappings()) .build(); return NodeProjections.single( @@ -261,14 +233,10 @@ private NodeProjections buildEstimateNodeProjections() { } private RelationshipProjections buildEstimateRelationshipProjections() { - if (cypherConfig.isFictitiousLoading()) { - relationshipEstimation = ImmutableEstimationResult.of(cypherConfig.relationshipCount(), 0); - } - var relationshipProjection = RelationshipProjection .builder() .type(ElementProjection.PROJECT_ALL) - .addAllProperties(getRelationshipEstimation().propertyMappings()) + .addAllProperties(relationshipEstimation.propertyMappings()) .build(); return RelationshipProjections.single( @@ -276,31 +244,4 @@ private RelationshipProjections buildEstimateRelationshipProjections() { relationshipProjection ); } - - @ValueClass - interface EstimationResult { - long estimatedRows(); - - long propertyCount(); - - @Value.Derived - default Map propertyTokens() { - return LongStream - .range(0, propertyCount()) - .boxed() - .collect(Collectors.toMap( - Object::toString, - property -> NO_SUCH_PROPERTY_KEY - )); - } - - @Value.Derived - default Collection propertyMappings() { - return LongStream - .range(0, propertyCount()) - .mapToObj(property -> PropertyMapping.of(Long.toString(property), DefaultValue.DEFAULT)) - .collect(Collectors.toList()); - } - - } } diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java new file mode 100644 index 00000000000..9933b3874d2 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.immutables.value.Value; +import org.neo4j.gds.PropertyMapping; +import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.api.DefaultValue; +import org.neo4j.gds.transaction.TransactionContext; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; +import static org.neo4j.internal.kernel.api.security.AccessMode.Static.READ; +import static org.neo4j.kernel.api.StatementConstants.NO_SUCH_PROPERTY_KEY; + +public class CypherQueryEstimator { + + private final TransactionContext context; + + CypherQueryEstimator(TransactionContext context) {this.context = context;} + + + EstimationResult getNodeEstimation(String nodeQuery) { + return runEstimationQuery(nodeQuery, NodeSubscriber.RESERVED_COLUMNS); + } + + EstimationResult getRelationshipEstimation(String relationshipQuery) { + return runEstimationQuery( + relationshipQuery, + RelationshipSubscriber.RESERVED_COLUMNS + ); + } + + private EstimationResult runEstimationQuery(String query, Collection reservedColumns) { + return context.withRestrictedAccess(READ).apply((tx, ktx) -> { + var explainQuery = formatWithLocale("EXPLAIN %s", query); + try (var result = tx.execute(explainQuery)) { + var estimatedRows = (Number) result.getExecutionPlanDescription().getArguments().get("EstimatedRows"); + + var propertyColumns = new ArrayList<>(result.columns()); + propertyColumns.removeAll(reservedColumns); + + return ImmutableEstimationResult.of(estimatedRows.longValue(), propertyColumns.size()); + } + }); + } + + @ValueClass + public + interface EstimationResult { + long estimatedRows(); + + long propertyCount(); + + @Value.Derived + default Map propertyTokens() { + return LongStream + .range(0, propertyCount()) + .boxed() + .collect(Collectors.toMap( + Object::toString, + property -> NO_SUCH_PROPERTY_KEY + )); + } + + @Value.Derived + default Collection propertyMappings() { + return LongStream + .range(0, propertyCount()) + .mapToObj(property -> PropertyMapping.of(Long.toString(property), DefaultValue.DEFAULT)) + .collect(Collectors.toList()); + } + + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java index b11c42c6921..e6cfedaf7ac 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/NativeFactory.java @@ -56,6 +56,7 @@ public final class NativeFactory extends CSRGraphStoreFactory { private final GraphProjectFromStoreConfig storeConfig; + private final ProgressTracker progressTracker; public NativeFactory( GraphProjectFromStoreConfig graphProjectConfig, @@ -79,6 +80,7 @@ public NativeFactory( ) { super(graphProjectConfig, ImmutableStaticCapabilities.of(true), loadingContext, graphDimensions); this.storeConfig = graphProjectConfig; + this.progressTracker = initProgressTracker(); } @Override @@ -250,8 +252,7 @@ private static void relationshipEstimationAfterLoading( }); } - @Override - protected ProgressTracker initProgressTracker() { + private ProgressTracker initProgressTracker() { long relationshipCount = graphProjectConfig .relationshipProjections() .projections() @@ -360,4 +361,9 @@ private RelationshipImportResult loadRelationships(IdMap idMap, int concurrency) progressTracker.endSubTask(); } } + + @Override + protected ProgressTracker progressTracker() { + return progressTracker; + } } diff --git a/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java b/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java index 27e7b90c565..e9b67c918fa 100644 --- a/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java @@ -217,26 +217,30 @@ void shouldLogProgressWithCypherLoading() { .databaseService(db) .graphName("graph") .nodeQuery("MATCH (n) RETURN id(n) AS id, coalesce(n.prop1, 42) AS prop1") - .relationshipQuery("MATCH (n)-[:REL1|REL2]->(m) RETURN id(n) AS source, id(m) AS target") + .relationshipQuery("MATCH (n)-[:REL1|REL2]-(m) RETURN id(n) AS source, id(m) AS target") .log(log) .build() .graph(); assertThat(log.getMessages(TestLog.INFO)) .extracting(removingThreadId()) - .contains( + .containsExactly( "Loading :: Start", "Loading :: Nodes :: Start", "Loading :: Nodes 33%", "Loading :: Nodes 66%", "Loading :: Nodes 100%", - "Loading :: Nodes :: Start", "Loading :: Nodes :: Finished", "Loading :: Relationships :: Start", + "Loading :: Relationships 8%", + "Loading :: Relationships 16%", "Loading :: Relationships 25%", + "Loading :: Relationships 33%", + "Loading :: Relationships 41%", "Loading :: Relationships 50%", - "Loading :: Relationships 75%", + "Loading :: Relationships 100%", "Loading :: Relationships :: Finished", - "Loading :: Finished" + "Loading :: Finished", + "Loading :: Actual memory usage of the loaded graph: 329 KiB" ); assertThat(log.getMessages(TestLog.DEBUG)).isEmpty(); diff --git a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java index dbeab231818..a736092effe 100644 --- a/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java +++ b/test-utils/src/main/java/org/neo4j/gds/gdl/GdlFactory.java @@ -155,11 +155,6 @@ public MemoryEstimation estimateMemoryUsageAfterLoading() { return MemoryEstimations.empty(); } - @Override - protected ProgressTracker initProgressTracker() { - return ProgressTracker.NULL_TRACKER; - } - @Override public CSRGraphStore build() { var nodes = loadNodes(); @@ -370,6 +365,11 @@ private double gdsValue(Element element, String propertyKey, Object gdlValue) { } } + @Override + protected ProgressTracker progressTracker() { + return ProgressTracker.NULL_TRACKER; + } + private static final class GraphDimensionsGdlReader { static GraphDimensions of(GDLHandler gdlHandler) { From d86ec022abbd7964a14d2fe7414d24fba03fdb4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 8 Mar 2023 16:20:36 +0100 Subject: [PATCH 272/400] Fix CypherFactory factory method for the EstimationCLI in the estimationcli we have fictitious loading and no transactionContext --- .../neo4j/gds/core/loading/CypherFactory.java | 27 ++++++++++--------- .../org/neo4j/gds/core/GraphLoaderTest.java | 11 ++------ 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index 17a5fb35b94..bb905f711c9 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -64,15 +64,18 @@ private static CypherFactory create( GraphLoaderContext loadingContext, @Nullable GraphDimensions dimensions ) { - var estimator = new CypherQueryEstimator(loadingContext.transactionContext().withRestrictedAccess(READ)); - EstimationResult nodeQueryEstimation = graphProjectConfig.isFictitiousLoading() - ? ImmutableEstimationResult.of(graphProjectConfig.nodeCount(), 0) - : estimator.getNodeEstimation(graphProjectConfig.nodeQuery()); + EstimationResult nodeEstimation; + EstimationResult relationEstimation; - EstimationResult relationshipQueryEstimation = graphProjectConfig.isFictitiousLoading() - ? ImmutableEstimationResult.of(graphProjectConfig.relationshipCount(), 0) - : estimator.getRelationshipEstimation(graphProjectConfig.relationshipQuery()); + if (graphProjectConfig.isFictitiousLoading()) { + nodeEstimation = ImmutableEstimationResult.of(graphProjectConfig.nodeCount(), 0); + relationEstimation = ImmutableEstimationResult.of(graphProjectConfig.relationshipCount(), 0); + } else { + var estimator = new CypherQueryEstimator(loadingContext.transactionContext().withRestrictedAccess(READ)); + nodeEstimation = estimator.getNodeEstimation(graphProjectConfig.nodeQuery()); + relationEstimation = estimator.getRelationshipEstimation(graphProjectConfig.relationshipQuery()); + } var dimBuilder = ImmutableGraphDimensions.builder(); @@ -81,17 +84,17 @@ private static CypherFactory create( } GraphDimensions dim = ImmutableGraphDimensions.builder() - .highestPossibleNodeCount(nodeQueryEstimation.estimatedRows()) - .nodeCount(nodeQueryEstimation.estimatedRows()) - .relCountUpperBound(relationshipQueryEstimation.estimatedRows()) + .highestPossibleNodeCount(nodeEstimation.estimatedRows()) + .nodeCount(nodeEstimation.estimatedRows()) + .relCountUpperBound(relationEstimation.estimatedRows()) .build(); return new CypherFactory( graphProjectConfig, loadingContext, dim, - nodeQueryEstimation, - relationshipQueryEstimation + nodeEstimation, + relationEstimation ); } diff --git a/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java b/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java index e9b67c918fa..b9e6ad819c6 100644 --- a/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java @@ -223,7 +223,7 @@ void shouldLogProgressWithCypherLoading() { .graph(); assertThat(log.getMessages(TestLog.INFO)) .extracting(removingThreadId()) - .containsExactly( + .contains( "Loading :: Start", "Loading :: Nodes :: Start", "Loading :: Nodes 33%", @@ -231,16 +231,9 @@ void shouldLogProgressWithCypherLoading() { "Loading :: Nodes 100%", "Loading :: Nodes :: Finished", "Loading :: Relationships :: Start", - "Loading :: Relationships 8%", - "Loading :: Relationships 16%", - "Loading :: Relationships 25%", - "Loading :: Relationships 33%", - "Loading :: Relationships 41%", - "Loading :: Relationships 50%", "Loading :: Relationships 100%", "Loading :: Relationships :: Finished", - "Loading :: Finished", - "Loading :: Actual memory usage of the loaded graph: 329 KiB" + "Loading :: Finished" ); assertThat(log.getMessages(TestLog.DEBUG)).isEmpty(); From cf09ef54d06fae243a8a87727a904ad78e0aa69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 8 Mar 2023 22:10:21 +0100 Subject: [PATCH 273/400] Add automated parts of 5.6 compat --- .../5.6/neo4j-kernel-adapter/build.gradle | 63 ++ .../gds/compat/_56/Neo4jProxyFactoryImpl.java | 44 + .../compat/_56/SettingProxyFactoryImpl.java | 44 + .../compat/_56/BoltTransactionRunnerImpl.java | 98 ++ .../gds/compat/_56/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_56/CompatAccessModeImpl.java | 44 + .../_56/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_56/CompatIndexQueryImpl.java | 31 + .../_56/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_56/CompositeNodeCursorImpl.java | 32 + .../gds/compat/_56/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../gds/compat/_56/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_56/Neo4jProxyImpl.java | 923 ++++++++++++++++++ .../compat/_56/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_56/PartitionedStoreScan.java | 58 ++ .../_56/ReferencePropertyReference.java | 49 + .../compat/_56/ScanBasedStoreScanImpl.java | 40 + .../compat/_56/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_56/SettingProxyImpl.java | 87 ++ .../org/neo4j/gds/compat/_56/TestLogImpl.java | 146 +++ .../compat/_56/VirtualRelationshipImpl.java | 41 + .../5.6/storage-engine-adapter/build.gradle | 65 ++ .../_56/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_56/InMemoryCountsStoreImpl.java | 110 +++ .../_56/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_56/InMemoryNodeCursor.java | 82 ++ .../_56/InMemoryNodePropertyCursor.java | 45 + .../compat/_56/InMemoryPropertyCursor.java | 71 ++ .../_56/InMemoryPropertySelectionImpl.java | 55 ++ .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_56/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_56/InMemoryStorageEngineFactory.java | 557 +++++++++++ .../compat/_56/InMemoryStorageEngineImpl.java | 294 ++++++ .../compat/_56/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_56/InMemoryStoreVersion.java | 68 ++ .../_56/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_56/InMemoryVersionCheck.java | 61 ++ .../_56/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_56/StorageEngineProxyImpl.java | 156 +++ .../InMemoryLogVersionRepository56.java | 71 ++ ...InMemoryStorageCommandReaderFactory56.java | 43 + .../InMemoryStorageReader56.java | 325 ++++++ gradle/dependencies.gradle | 1 + 47 files changed, 4970 insertions(+) create mode 100644 compatibility/5.6/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableProcedureImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatAccessModeImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatIndexQueryImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/PartitionedStoreScan.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ReferencePropertyReference.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/TestLogImpl.java create mode 100644 compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/VirtualRelationshipImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodeCursor.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertyCursor.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStoreVersion.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryVersionCheck.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository56.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory56.java create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader56.java diff --git a/compatibility/5.6/neo4j-kernel-adapter/build.gradle b/compatibility/5.6/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..28100243536 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.6' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.6' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.6' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.6' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.6' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.6' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.6' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..5f0279f2fc5 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.6 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.6 (placeholder)"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..98d6a61dca1 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.6 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.6 (placeholder)"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/BoltTransactionRunnerImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..974956d0662 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/BoltTransactionRunnerImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.bookmark.Bookmark; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.v41.message.request.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableProcedureImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableProcedureImpl.java new file mode 100644 index 00000000000..bd0b3c0a07a --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableUserAggregationFunctionImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..b1f07487e92 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatAccessModeImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatAccessModeImpl.java new file mode 100644 index 00000000000..230ae244460 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatGraphDatabaseAPIImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..b99d39e27c0 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatIndexQueryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..2e06af57725 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatUsernameAuthSubjectImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..3397d8d71a0 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompositeNodeCursorImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..b4d73fca9f6 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseLayoutImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..2173de12ab1 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..65ad6f6a250 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..f064a42e37c --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_6; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.6"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java new file mode 100644 index 00000000000..125ce3a89a9 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -0,0 +1,923 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public long pageCacheMemory() { + return pageCacheMemory.orElseGet(Configuration.super::pageCacheMemory); + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/NodeLabelIndexLookupImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..0d9d2a3ae36 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/PartitionedStoreScan.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/PartitionedStoreScan.java new file mode 100644 index 00000000000..a3a52ce2266 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ReferencePropertyReference.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ReferencePropertyReference.java new file mode 100644 index 00000000000..303d8e528cc --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ScanBasedStoreScanImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..6da9397945f --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..af413d52556 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_6; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.6"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyImpl.java new file mode 100644 index 00000000000..ffd55552a5f --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/TestLogImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/TestLogImpl.java new file mode 100644 index 00000000000..e7ddb76d10e --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/VirtualRelationshipImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..84a53954543 --- /dev/null +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/build.gradle b/compatibility/5.6/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..0f28b9de440 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/build.gradle @@ -0,0 +1,65 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.6' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.6' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.6' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.6' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.6' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.6' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..5e6ab3d3d07 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.6"; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCommandCreationContextImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..2527d95468e --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCountsStoreImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..224369a1f17 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryCountsStoreImpl.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryMetaDataProviderImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..7460d90a9ca --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository56; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository56 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository56(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodeCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodeCursor.java new file mode 100644 index 00000000000..d1e9ac4939f --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodePropertyCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..b183e95d4eb --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertyCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..f4d8b3034d4 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertySelectionImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..d0c4d11bae8 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipPropertyCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..4b0895b48c7 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipScanCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..9efc1b9563d --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipTraversalCursor.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..2b2ad14d00c --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..c95775df36d --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java @@ -0,0 +1,557 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory56; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-rc"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_6, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory56.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..dd36aca80ee --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineImpl.java @@ -0,0 +1,294 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader56; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private final StorageEngineIndexingBehaviour indexingBehaviour = () -> false; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return indexingBehaviour; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader56(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageLocksImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..62bb86eb377 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStoreVersion.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStoreVersion.java new file mode 100644 index 00000000000..8f057a608a7 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryTransactionIdStoreImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..fb77020f557 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryVersionCheck.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryVersionCheck.java new file mode 100644 index 00000000000..e9a743e106f --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._56.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..4d03274003a --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_6; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.6"; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..b5a1b5a4ffc --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader56; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader56(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository56.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository56.java new file mode 100644 index 00000000000..0030425d089 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository56.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository56 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository56() { + this(0, 0); + } + + private InMemoryLogVersionRepository56(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory56.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory56.java new file mode 100644 index 00000000000..188dfd83144 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory56.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory56 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory56(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader56.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader56.java new file mode 100644 index 00000000000..921062f5176 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader56.java @@ -0,0 +1,325 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._56.InMemoryNodeCursor; +import org.neo4j.gds.compat._56.InMemoryPropertyCursor; +import org.neo4j.gds.compat._56.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._56.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader56 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader56( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index a76fa5d71c4..aba1af8db72 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -6,6 +6,7 @@ ext { '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), '5.4': properties.getOrDefault('neo4jVersion54', '5.4.0'), '5.5': properties.getOrDefault('neo4jVersion55', '5.5.0'), + '5.6': properties.getOrDefault('neo4jVersion56', '5.6.0'), ] neo4jDefault = neos.'4.4' From 46f8794a1df3ed788af896173b8302998248ec3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 8 Mar 2023 23:26:41 +0100 Subject: [PATCH 274/400] Add manual parts of 5.6 compat --- .../java/org/neo4j/gds/compat/Neo4jVersion.java | 7 ++++++- .../java/org/neo4j/gds/compat/Neo4jVersionTest.java | 4 +++- .../test/java/org/neo4j/gds/SysInfoProcTest.java | 13 +++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index aa9473fe673..473aa513dd3 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -33,6 +33,7 @@ public enum Neo4jVersion { V_5_3, V_5_4, V_5_5, + V_5_6, V_RC; @Override @@ -50,6 +51,8 @@ public String toString() { return "5.4"; case V_5_5: return "5.5"; + case V_5_6: + return "5.6"; case V_RC: return "rc"; default: @@ -59,7 +62,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 6); + return ImmutableMajorMinorVersion.of(5, 7); } String version = toString(); var subVersions = version.split("\\."); @@ -132,6 +135,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 5) { return Neo4jVersion.V_5_5; } else if (minorVersion == 6) { + return Neo4jVersion.V_5_6; + } else if (minorVersion == 7) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 0ee6b822964..0989f6bad2b 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -46,7 +46,8 @@ class Neo4jVersionTest { "5.3.0, V_5_3", "5.4.0, V_5_4", "5.5.0, V_5_5", - "5.6.0, V_RC", + "5.6.0, V_5_6", + "5.7.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); @@ -84,6 +85,7 @@ void shouldNotRespectVersionOverride() { "5.3.0, 5, 3", "5.4.0, 5, 4", "5.5.0, 5, 5", + "5.6.0, 5, 6", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index d07b22c5487..d2c3b78231e 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -76,6 +76,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.5", "Neo4j Settings 5.5 (placeholder)", + "Neo4j 5.6", + "Neo4j 5.6 (placeholder)", + "Neo4j Settings 5.6", + "Neo4j Settings 5.6 (placeholder)", + "Neo4j RC", "Neo4j RC (placeholder)", "Neo4j Settings RC", @@ -171,6 +176,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.5" ); break; + case V_5_6: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.6 (placeholder)", + "Neo4j Settings 5.6", + "Neo4j 5.6 (placeholder)", + "Neo4j 5.6" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC (placeholder)", From 1abe53e290018f244b59a92911018f16352654f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 9 Mar 2023 12:17:32 +0100 Subject: [PATCH 275/400] Move all dimensions computation to the factory --- .../neo4j/gds/core/loading/CypherFactory.java | 77 +++++++++++-------- .../org/neo4j/gds/core/GraphLoaderTest.java | 13 +++- 2 files changed, 58 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java index bb905f711c9..9e9199fca9b 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherFactory.java @@ -23,10 +23,12 @@ import org.neo4j.gds.NodeLabel; import org.neo4j.gds.NodeProjection; import org.neo4j.gds.NodeProjections; +import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.RelationshipProjection; import org.neo4j.gds.RelationshipProjections; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.CSRGraphStoreFactory; +import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.api.GraphLoaderContext; import org.neo4j.gds.config.GraphProjectFromCypherConfig; import org.neo4j.gds.core.GraphDimensions; @@ -39,7 +41,10 @@ import org.neo4j.gds.core.utils.warnings.EmptyUserLogRegistryFactory; import org.neo4j.gds.transaction.TransactionContext; -import javax.annotation.Nullable; +import java.util.Collection; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.LongStream; import static org.neo4j.gds.core.loading.CypherQueryEstimator.EstimationResult; import static org.neo4j.internal.kernel.api.security.AccessMode.Static.READ; @@ -47,22 +52,22 @@ public final class CypherFactory extends CSRGraphStoreFactory { private final GraphProjectFromCypherConfig cypherConfig; - private final EstimationResult nodeEstimation; - private final EstimationResult relationshipEstimation; + private final long numberOfNodeProperties; + private final long numberOfRelationshipProperties; private final ProgressTracker progressTracker; public static CypherFactory createWithBaseDimensions(GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext, GraphDimensions graphDimensions) { - return create(graphProjectConfig, loadingContext, graphDimensions); + return create(graphProjectConfig, loadingContext, Optional.of(graphDimensions)); } public static CypherFactory createWithDerivedDimensions(GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext) { - return create(graphProjectConfig, loadingContext, null); + return create(graphProjectConfig, loadingContext, Optional.empty()); } private static CypherFactory create( GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext, - @Nullable GraphDimensions dimensions + Optional baseDimensions ) { EstimationResult nodeEstimation; @@ -79,22 +84,32 @@ private static CypherFactory create( var dimBuilder = ImmutableGraphDimensions.builder(); - if (dimensions != null) { - dimBuilder.from(dimensions); - } + baseDimensions.ifPresent(dimBuilder::from); + + long highestPossibleNodeCount = Math.max(baseDimensions + .map(GraphDimensions::highestPossibleNodeCount) + .orElse(-1L), nodeEstimation.estimatedRows()); + long nodeCount = Math.max( + baseDimensions.map(GraphDimensions::nodeCount).orElse(-1L), + nodeEstimation.estimatedRows() + ); + long relCountUpperBound = Math.max( + baseDimensions.map(GraphDimensions::relCountUpperBound).orElse(-1L), + relationEstimation.estimatedRows() + ); - GraphDimensions dim = ImmutableGraphDimensions.builder() - .highestPossibleNodeCount(nodeEstimation.estimatedRows()) - .nodeCount(nodeEstimation.estimatedRows()) - .relCountUpperBound(relationEstimation.estimatedRows()) + GraphDimensions dim = dimBuilder + .highestPossibleNodeCount(highestPossibleNodeCount) + .nodeCount(nodeCount) + .relCountUpperBound(relCountUpperBound) .build(); return new CypherFactory( graphProjectConfig, loadingContext, dim, - nodeEstimation, - relationEstimation + nodeEstimation.propertyCount(), + relationEstimation.propertyCount() ); } @@ -102,16 +117,15 @@ private CypherFactory( GraphProjectFromCypherConfig graphProjectConfig, GraphLoaderContext loadingContext, GraphDimensions graphDimensions, - EstimationResult nodeEstimation, - EstimationResult relationshipEstimation - + long estimatedNumberOfNodeProperties, + long estimatedNumberOfRelProperties ) { // TODO: need to pass capabilities from outside? super(graphProjectConfig, ImmutableStaticCapabilities.of(true), loadingContext, graphDimensions); this.cypherConfig = graphProjectConfig; - this.nodeEstimation = nodeEstimation; - this.relationshipEstimation = relationshipEstimation; + this.numberOfNodeProperties = estimatedNumberOfNodeProperties; + this.numberOfRelationshipProperties = estimatedNumberOfRelProperties; this.progressTracker = initProgressTracker(); } @@ -140,12 +154,7 @@ public MemoryEstimation estimateMemoryUsageAfterLoading() { @Override public GraphDimensions estimationDimensions() { - return ImmutableGraphDimensions.builder() - .from(dimensions) - .highestPossibleNodeCount(Math.max(dimensions.highestPossibleNodeCount(), nodeEstimation.estimatedRows())) - .nodeCount(Math.max(dimensions.nodeCount(), nodeEstimation.estimatedRows())) - .relCountUpperBound(Math.max(dimensions.relCountUpperBound(), relationshipEstimation.estimatedRows())) - .build(); + return dimensions; } @Override @@ -190,11 +199,10 @@ public CSRGraphStore build() { } private ProgressTracker initProgressTracker() { - var estimatedDimensions = estimationDimensions(); var task = Tasks.task( "Loading", - Tasks.leaf("Nodes", estimatedDimensions.highestPossibleNodeCount()), - Tasks.leaf("Relationships", estimatedDimensions.relCountUpperBound()) + Tasks.leaf("Nodes", dimensions.highestPossibleNodeCount()), + Tasks.leaf("Relationships", dimensions.relCountUpperBound()) ); if (graphProjectConfig.logProgress()) { @@ -226,7 +234,7 @@ private NodeProjections buildEstimateNodeProjections() { var nodeProjection = NodeProjection .builder() .label(ElementProjection.PROJECT_ALL) - .addAllProperties(nodeEstimation.propertyMappings()) + .addAllProperties(propertyMappings(numberOfNodeProperties)) .build(); return NodeProjections.single( @@ -239,7 +247,7 @@ private RelationshipProjections buildEstimateRelationshipProjections() { var relationshipProjection = RelationshipProjection .builder() .type(ElementProjection.PROJECT_ALL) - .addAllProperties(relationshipEstimation.propertyMappings()) + .addAllProperties(propertyMappings(numberOfRelationshipProperties)) .build(); return RelationshipProjections.single( @@ -247,4 +255,11 @@ private RelationshipProjections buildEstimateRelationshipProjections() { relationshipProjection ); } + + private static Collection propertyMappings(long propertyCount) { + return LongStream + .range(0, propertyCount) + .mapToObj(property -> PropertyMapping.of(Long.toString(property), DefaultValue.DEFAULT)) + .collect(Collectors.toList()); + } } diff --git a/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java b/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java index b9e6ad819c6..5caefb17c78 100644 --- a/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java +++ b/core/src/test/java/org/neo4j/gds/core/GraphLoaderTest.java @@ -47,6 +47,8 @@ import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -212,6 +214,7 @@ public void shouldTrackProgressWithNativeLoadingUsingIndex() { @Test void shouldLogProgressWithCypherLoading() { + var progressRegex = Pattern.compile("(\\d+)%$"); var log = Neo4jProxy.testLog(); new CypherLoaderBuilder() .databaseService(db) @@ -234,7 +237,15 @@ void shouldLogProgressWithCypherLoading() { "Loading :: Relationships 100%", "Loading :: Relationships :: Finished", "Loading :: Finished" - ); + ) + .noneMatch(message -> { + Matcher matcher = progressRegex.matcher(message); + if (matcher.find()) { + int progress = Integer.parseInt(matcher.group(1)); + return progress > 100; + } + return false; + }); assertThat(log.getMessages(TestLog.DEBUG)).isEmpty(); } From 24235252402847e2da42c1d7a8f1e2704214aaf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 9 Mar 2023 13:20:18 +0100 Subject: [PATCH 276/400] Test CypherQueryEstimator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Max Kießling --- .../core/loading/CypherQueryEstimator.java | 27 -------- .../loading/CypherQueryEstimatorTest.java | 68 +++++++++++++++++++ 2 files changed, 68 insertions(+), 27 deletions(-) create mode 100644 core/src/test/java/org/neo4j/gds/core/loading/CypherQueryEstimatorTest.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java index 9933b3874d2..8de1954c8de 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/CypherQueryEstimator.java @@ -19,21 +19,14 @@ */ package org.neo4j.gds.core.loading; -import org.immutables.value.Value; -import org.neo4j.gds.PropertyMapping; import org.neo4j.gds.annotation.ValueClass; -import org.neo4j.gds.api.DefaultValue; import org.neo4j.gds.transaction.TransactionContext; import java.util.ArrayList; import java.util.Collection; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.LongStream; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; import static org.neo4j.internal.kernel.api.security.AccessMode.Static.READ; -import static org.neo4j.kernel.api.StatementConstants.NO_SUCH_PROPERTY_KEY; public class CypherQueryEstimator { @@ -73,25 +66,5 @@ interface EstimationResult { long estimatedRows(); long propertyCount(); - - @Value.Derived - default Map propertyTokens() { - return LongStream - .range(0, propertyCount()) - .boxed() - .collect(Collectors.toMap( - Object::toString, - property -> NO_SUCH_PROPERTY_KEY - )); - } - - @Value.Derived - default Collection propertyMappings() { - return LongStream - .range(0, propertyCount()) - .mapToObj(property -> PropertyMapping.of(Long.toString(property), DefaultValue.DEFAULT)) - .collect(Collectors.toList()); - } - } } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/CypherQueryEstimatorTest.java b/core/src/test/java/org/neo4j/gds/core/loading/CypherQueryEstimatorTest.java new file mode 100644 index 00000000000..0dcc4f1af35 --- /dev/null +++ b/core/src/test/java/org/neo4j/gds/core/loading/CypherQueryEstimatorTest.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.BaseTest; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.extension.Neo4jGraph; +import org.neo4j.gds.transaction.TransactionContext; + +import static org.assertj.core.api.Assertions.assertThat; + +class CypherQueryEstimatorTest extends BaseTest { + + @Neo4jGraph + private static final String DB_CYPHER = + "CREATE" + + " (a1:A {property: 33, a: 33})" + + ", (a2:A {property: 33, a: 33})" + + ", (b:B {property: 42, b: 42})" + + ", (a1)-[:T1 {property1: 42, property2: 1337}]->(b)" + + ", (a2)-[:T2 {property1: 43}]->(b)" + + ", (a2)-[:T2 {property1: 43}]->(a1)"; + + @Test + void estimateNodes() { + GraphDatabaseApiProxy.runInTransaction(db, tx -> { + CypherQueryEstimator estimator = new CypherQueryEstimator(TransactionContext.of(db, tx)); + + var estimation = estimator.getNodeEstimation( + "MATCH (n) RETURN id(n) AS id, labels(n) AS labels, n.property AS score"); + + // EXPLAIN seems to overestimate the nodeCount here + assertThat(estimation).isEqualTo(ImmutableEstimationResult.of(10, 1)); + }); + } + + @Test + void estimateRelationships() { + GraphDatabaseApiProxy.runInTransaction(db, tx -> { + CypherQueryEstimator estimator = new CypherQueryEstimator(TransactionContext.of(db, tx)); + + var estimation = estimator.getRelationshipEstimation( + "MATCH (n)-[r]-(m) RETURN id(n) AS source, m AS target, r.property1 AS score, type(r) AS type"); + + assertThat(estimation).isEqualTo(ImmutableEstimationResult.of(6, 1)); + }); + } + + +} From 990ecbc6f991f0e48f8cb892f5402f1353592581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 9 Mar 2023 17:38:34 +0100 Subject: [PATCH 277/400] Document 5.6 support --- README.adoc | 5 +++-- .../ROOT/pages/installation/supported-neo4j-versions.adoc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index 0900724a380..64d82e539e1 100644 --- a/README.adoc +++ b/README.adoc @@ -77,19 +77,20 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library .6+<.^|GDS 2.2.x |Neo4j 4.3.15 - 4.3.23 -.11+.^|Java 11 & Java 17 +.12+.^|Java 11 & Java 17 |Neo4j 4.4.9 - 4.4.16 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 |Neo4j 5.4.0 -.6+<.^|GDS 2.3.x +.7+<.^|GDS 2.3.x |Neo4j 4.4.9 - 4.4.18 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 |Neo4j 5.4.0 |Neo4j 5.5.0 +|Neo4j 5.6.0 |=== NOTE: Preview releases are not automatically made available in Neo4j Desktop. They need to be installed manually. diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index a70ac5056b9..f0c38f388c3 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,6 +10,7 @@ Time to upgrade! [opts=header] |=== | Neo4j version | Neo4j Graph Data Science +| `5.5` | `2.3.2 or later` | `5.5` | `2.3.1 or later` | `5.4` | `2.3`, `2.2.7 or later` footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] | `5.3` | `2.3`, `2.2.6 or later` footnote:eol[] From 38044b8afd01a1c141210b91660f9a23a277c1f6 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Fri, 10 Mar 2023 07:26:41 +0100 Subject: [PATCH 278/400] Demonstrate and fix bug Co-authored-by: Veselin Nikolov --- .../neo4j/gds/influenceMaximization/CELF.java | 11 ++-- .../impl/influenceMaximization/CelfTest.java | 53 +++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/influenceMaximization/CelfTest.java diff --git a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java index 528f89336a5..562f71b9228 100644 --- a/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java +++ b/algo/src/main/java/org/neo4j/gds/influenceMaximization/CELF.java @@ -95,15 +95,15 @@ protected boolean lessThan(long a, long b) { public LongDoubleScatterMap compute() { //Find the first node with greedy algorithm progressTracker.beginSubTask(); - greedyPart(); + var firstSeedNode = greedyPart(); //Find the next k-1 nodes using the list-sorting procedure - lazyForwardPart(); + lazyForwardPart(firstSeedNode); progressTracker.endSubTask(); return seedSetNodes; } - private void greedyPart() { + private long greedyPart() { HugeDoubleArray singleSpreadArray = HugeDoubleArray.newArray(graph.nodeCount()); progressTracker.beginSubTask(graph.nodeCount()); var tasks = PartitionUtils.rangePartition( @@ -136,15 +136,16 @@ private void greedyPart() { gain = spreads.cost(highestNode); spreads.pop(); seedSetNodes.put(highestNode, gain); + return highestNode; } - private void lazyForwardPart() { + private void lazyForwardPart(long firstSeedNode) { var independentCascade = ICLazyForwardMC.create( graph, propagationProbability, monteCarloSimulations, - seedSetNodes.keys[0], + firstSeedNode, (int) seedSetCount, concurrency, executorService, diff --git a/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/influenceMaximization/CelfTest.java b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/influenceMaximization/CelfTest.java new file mode 100644 index 00000000000..f35c40dadce --- /dev/null +++ b/alpha/alpha-algo/src/test/java/org/neo4j/gds/impl/influenceMaximization/CelfTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.impl.influenceMaximization; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.api.schema.Direction; +import org.neo4j.gds.beta.generator.RandomGraphGenerator; +import org.neo4j.gds.beta.generator.RelationshipDistribution; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.influenceMaximization.CELF; + +import static org.assertj.core.api.Assertions.assertThat; + + class CelfTest { + + @ParameterizedTest + @ValueSource(ints = {2, 7}) + void shouldNotReturnNegativeGains(int seedSize) { + var graph = RandomGraphGenerator + .builder() + .averageDegree(5) + .relationshipDistribution(RelationshipDistribution.POWER_LAW) + .direction(Direction.DIRECTED) + .nodeCount(60) + .seed(42) + .build() + .generate(); + + var celf = new CELF(graph, seedSize, 0.1, 3, Pools.DEFAULT, 1, 10, 5, ProgressTracker.NULL_TRACKER).compute(); + for (var a : celf) { + assertThat(a.value).isNotNegative(); + } + } +} From 3773da2f8535ef31b354dba7eb323afa20bec55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Fri, 10 Mar 2023 15:32:34 +0100 Subject: [PATCH 279/400] Update Aura build number to 15 Co-Authored-By: Mats Rydberg --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 612d3ba2a1a..65ecbe32df7 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.2' - gdsAuraVersion = '14' + gdsAuraVersion = '15' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 69f4e02a83f469373d6b57c44b6f964d46033933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Tue, 28 Feb 2023 16:42:36 +0100 Subject: [PATCH 280/400] Implement drop cypher db procedure --- .../neo4j/gds/cypher/DropCypherDbProc.java | 90 +++++++++++++++++++ .../gds/cypher/GraphCreateCypherDbProc.java | 5 +- .../neo4j/gds/OpenGdsProcedureSmokeTest.java | 3 +- 3 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java new file mode 100644 index 00000000000..8d4635f7e99 --- /dev/null +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.cypher; + +import org.apache.commons.lang3.mutable.MutableLong; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseNotFoundException; +import org.neo4j.gds.BaseProc; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.core.utils.ProgressTimer; +import org.neo4j.gds.executor.ProcPreconditions; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.procedure.Description; +import org.neo4j.procedure.Name; +import org.neo4j.procedure.Procedure; + +import java.util.stream.Stream; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; +import static org.neo4j.procedure.Mode.WRITE; + +public class DropCypherDbProc extends BaseProc { + + private static final String DESCRIPTION = "Drop a database backed by an in-memory graph"; + + @Procedure(name = "gds.alpha.drop.cypherdb", mode = WRITE) + @Description(DESCRIPTION) + public Stream dropDb( + @Name(value = "dbName") String dbName + ) { + ProcPreconditions.check(); + + DropCypherDbResult result = runWithExceptionLogging( + "Drop in-memory Cypher database failed", + () -> { + GraphCreateCypherDbProc.validateNeo4jEnterpriseEdition(databaseService); + var dbms = GraphDatabaseApiProxy.resolveDependency(databaseService, DatabaseManagementService.class); + validateDatabaseName(dbName, dbms); + var dropMillis = new MutableLong(0); + try (var ignored = ProgressTimer.start(dropMillis::setValue)) { + dbms.dropDatabase(dbName); + } + return new DropCypherDbResult(dbName, dropMillis.getValue()); + } + ); + + return Stream.of(result); + } + + private static void validateDatabaseName(String dbName, DatabaseManagementService dbms) { + if (!dbms.listDatabases().contains(dbName)) { + throw new DatabaseNotFoundException(formatWithLocale("A database with name `%s` does not exist", dbName)); + } + + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(dbName); + if (graphName == null) { + throw new IllegalArgumentException(formatWithLocale( + "Database with name `%s` is not an in-memory database", + dbName + )); + } + } + + public static class DropCypherDbResult { + public final String dbName; + public final long dropMillis; + + public DropCypherDbResult(String dbName, long dropMillis) { + this.dbName = dbName; + this.dropMillis = dropMillis; + } + } +} diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/GraphCreateCypherDbProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/GraphCreateCypherDbProc.java index 6402f4d0607..aff8e66825f 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/GraphCreateCypherDbProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/GraphCreateCypherDbProc.java @@ -27,6 +27,7 @@ import org.neo4j.gds.compat.StorageEngineProxy; import org.neo4j.gds.core.utils.ProgressTimer; import org.neo4j.gds.storageengine.InMemoryDatabaseCreator; +import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; import org.neo4j.procedure.Procedure; @@ -52,7 +53,7 @@ public Stream createDb( CreateCypherDbResult result = runWithExceptionLogging( "In-memory Cypher database creation failed", () -> { - validateNeo4jEnterpriseEdition(); + validateNeo4jEnterpriseEdition(databaseService); MutableLong createMillis = new MutableLong(0); try (ProgressTimer ignored = ProgressTimer.start(createMillis::setValue)) { InMemoryDatabaseCreator.createDatabase(databaseService, username(), graphName, dbName); @@ -78,7 +79,7 @@ public CreateCypherDbResult(String dbName, String graphName, long createMillis) } } - private void validateNeo4jEnterpriseEdition() { + static void validateNeo4jEnterpriseEdition(GraphDatabaseService databaseService) { var edition = StorageEngineProxy.dbmsEdition(databaseService); if (!(edition == Edition.ENTERPRISE)) { throw new DatabaseManagementException(formatWithLocale( diff --git a/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java b/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java index 15b55c5b63c..741656ba776 100644 --- a/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java +++ b/open-packaging/src/test/java/org/neo4j/gds/OpenGdsProcedureSmokeTest.java @@ -52,6 +52,7 @@ class OpenGdsProcedureSmokeTest extends BaseProcTest { "gds.alpha.graph.sample.rwr", "gds.alpha.create.cypherdb", + "gds.alpha.drop.cypherdb", "gds.alpha.allShortestPaths.stream", @@ -532,7 +533,7 @@ void countShouldMatch() { ); // If you find yourself updating this count, please also update the count in SmokeTest.kt - int expectedCount = 377; + int expectedCount = 378; assertEquals( expectedCount, registeredProcedures.size(), From 7d354070ad532c88c80378b85e5bf9f67ecee592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Thu, 2 Mar 2023 10:58:57 +0100 Subject: [PATCH 281/400] Document cypher db drop proc --- .../management-ops/create-cypher-db.adoc | 36 +++++++++++++++---- .../additional-operation-references.adoc | 1 + 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/create-cypher-db.adoc b/doc/modules/ROOT/pages/management-ops/create-cypher-db.adoc index 309948fef95..27decb76254 100644 --- a/doc/modules/ROOT/pages/management-ops/create-cypher-db.adoc +++ b/doc/modules/ROOT/pages/management-ops/create-cypher-db.adoc @@ -1,4 +1,4 @@ -[[create-cypher-db]] +[[cypher-on-gds]] = Cypher on GDS graph :description: This chapter explains how to execute Cypher queries on named graphs in the Neo4j Graph Data Science library. @@ -23,14 +23,13 @@ That database will then use data from the projected graph as compared to the sto Although it is possible to execute arbitrary Cypher queries on the database created by the `gds.alpha.create.cypherdb` procedure, not every aspect of Cypher is implemented yet. Some known limitations are listed below: -* Dropping the newly created database -** Restarting the DBMS will remove the database instead -* Writes -** All queries that attempt to write things, such as nodes, properties or labels, will fail +* Some writes will fail +** Creating new nodes and adding node labels +** Everything related to relationships [[create-cypher-db-syntax]] -== Syntax +== Create database syntax [.create-cypher-db-syntax] -- @@ -157,3 +156,28 @@ MATCH (n:Person)-[:KNOWS]->(m:Person) RETURN n.age AS age1, m.age AS age2 |=== We can see that the returned ages correspond to the structure of the original graph. + + +[[drop-cypher-db]] +== Dropping a GDS database + +As described above, in-memory GDS databases are impermanent and will be removed when the DBMS is shut down. +If we need to drop the GDS database earlier, there are 2 ways to achieve this: + 1. Using an administrative cypher command against the system database (`DROP DATABASE `) + 2. Using the <> procedure + +[[drop-cypher-db-procedure-syntax]] +=== Drop database syntax + +[.drop-cypher-db-syntax] +-- +[source, cypher, role=noplay] +---- +CALL gds.alpha.drop.cypherdb( + dbName: String +) +YIELD + dbName: String, + dropMillis: Integer +---- +-- diff --git a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc index 72aedc27dc1..5753bcc6277 100644 --- a/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/additional-operation-references.adoc @@ -22,6 +22,7 @@ | xref:alpha-algorithms/one-hot-encoding.adoc[One Hot Encoding] | `_gds.alpha.ml.oneHotEncoding_` | xref:common-usage/debug-sysinfo.adoc[Status of the system] | `gds.debug.sysInfo` | xref:management-ops/create-cypher-db.adoc[Create an impermanent database backed by a projected graph] | `gds.alpha.create.cypherdb` +| xref:management-ops/create-cypher-db.adoc#drop-cypher-db[Drop an impermanent database backed by a projected graph] | `gds.alpha.drop.cypherdb` | xref:common-usage/monitoring-system.adoc[Get an overview of the system's workload and available resources] | `gds.alpha.systemMonitor` | xref:management-ops/backup-restore.adoc[Back-up graphs and models to disk] | `gds.alpha.backup` | xref:management-ops/backup-restore.adoc[Restore persisted graphs and models to memory] | `gds.alpha.restore` From 2749d289f00c1b9e873871f69edd293195639da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Tue, 14 Mar 2023 11:10:29 +0100 Subject: [PATCH 282/400] Fix ProcPreconditions import --- .../src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java b/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java index 8d4635f7e99..68f5cf91875 100644 --- a/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java +++ b/alpha/alpha-proc/src/main/java/org/neo4j/gds/cypher/DropCypherDbProc.java @@ -23,9 +23,9 @@ import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.dbms.api.DatabaseNotFoundException; import org.neo4j.gds.BaseProc; +import org.neo4j.gds.ProcPreconditions; import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.core.utils.ProgressTimer; -import org.neo4j.gds.executor.ProcPreconditions; import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; import org.neo4j.procedure.Description; import org.neo4j.procedure.Name; From 05684b30b47e9c9d7cf7d9d4276251b642ca2ae2 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Wed, 15 Mar 2023 15:56:44 +0000 Subject: [PATCH 283/400] Disable assert for flaky Wcc sampled doc test There is so much text around the example that explains the result. --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index 7dd79267831..dc8a970fe48 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -626,7 +626,7 @@ CALL gds.graph.project( The following query is identical to the stream example in the xref:algorithms/wcc.adoc#algorithms-wcc-examples-seeding[previous section]. This time, we execute WCC on `myIndexedGraph` which will allow the algorithm to use the sampled strategy. -[role=query-example] +[role=query-example, no-result=true] -- .The following will run the algorithm with sampled strategy and stream results: [source, cypher, role=noplay] From f6320f4db4e83882c4cf1d22708c4c6c0788d4c2 Mon Sep 17 00:00:00 2001 From: Ioannis Pan <74839024+IoannisPanagiotas@users.noreply.github.com> Date: Fri, 17 Mar 2023 11:11:08 +0100 Subject: [PATCH 284/400] Fix title 2.3 --- doc/modules/ROOT/pages/algorithms/yens.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/yens.adoc b/doc/modules/ROOT/pages/algorithms/yens.adoc index 42558b7c059..8e5f9a37af8 100644 --- a/doc/modules/ROOT/pages/algorithms/yens.adoc +++ b/doc/modules/ROOT/pages/algorithms/yens.adoc @@ -1,5 +1,5 @@ [[algorithms-yens]] -= Yen's algorithm Shortest Path += Yen's Shortest Path algorithm :description: This section describes the Yen's Shortest Path algorithm in the Neo4j Graph Data Science library. :entity: source-target-pair :result: shortest path From 2d78e6696f08409cfcc9def2a69641226a13d713 Mon Sep 17 00:00:00 2001 From: Jacob Sznajdman Date: Mon, 13 Mar 2023 12:20:47 +0100 Subject: [PATCH 285/400] Fix bug in SubGraph the neighbor adjacency is invoked by (mapped) nodeIds, but was stored using batch offsets due to possible duplicates of node ids in the batch, the offset is not neccessarily the same as the mapped id --- .../graphsage/GraphSageModelTrainerTest.java | 78 +++++++++++++------ .../GraphSageTrainAlgorithmFactoryTest.java | 8 +- .../neo4j/gds/ml/core/subgraph/SubGraph.java | 20 +++-- .../ml/core/subgraph/SubGraphBuilderTest.java | 2 +- 4 files changed, 68 insertions(+), 40 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java index afb6bbd3373..145972a1572 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/GraphSageModelTrainerTest.java @@ -26,10 +26,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; import org.neo4j.gds.Orientation; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.beta.generator.PropertyProducer; +import org.neo4j.gds.beta.generator.RandomGraphGenerator; +import org.neo4j.gds.beta.generator.RelationshipDistribution; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.utils.paged.HugeObjectArray; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -77,6 +81,7 @@ class GraphSageModelTrainerTest { private Graph unweightedGraph; @Inject private Graph arrayGraph; + private HugeObjectArray features; private GraphSageTrainConfigImpl.Builder configBuilder; @@ -97,6 +102,53 @@ void setUp() { .embeddingDimension(EMBEDDING_DIMENSION); } + // This reproduced bug in https://trello.com/c/BQ3e12K3/7826-250-graphsage-returns-nan-when-using-relationship-weights + // https://github.com/neo4j/graph-data-science/issues/250 + @ParameterizedTest + @EnumSource(AggregatorType.class) + void trainsWithRelationshipWeight(AggregatorType aggregatorType) { + + var config = GraphSageTrainConfigImpl.builder() + .randomSeed(42L) + .batchSize(100) + .relationshipWeightProperty("p") + .embeddingDimension(2) + .aggregator(aggregatorType) + .activationFunction(ActivationFunction.SIGMOID) + .featureProperties(List.of("features")) + .modelName("model") + .modelUser("") + .build(); + + var trainModel = new GraphSageModelTrainer(config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + + int nodeCount = 5_000; + var bigGraph = RandomGraphGenerator + .builder() + .nodeCount(nodeCount) + .averageDegree(1) + .relationshipDistribution(RelationshipDistribution.UNIFORM) + .nodePropertyProducer(PropertyProducer.randomEmbedding("features", 1, -100, 100)) + .relationshipPropertyProducer(PropertyProducer.fixedDouble("p", 0.5)) + .seed(42L) + .build() + .generate(); + + features = HugeObjectArray.newArray(double[].class, nodeCount); + + LongStream.range(0, nodeCount).forEach(n -> features.set(n, bigGraph.nodeProperties().get("features").doubleArrayValue(n))); + + GraphSageModelTrainer.ModelTrainResult result = trainModel.train( + bigGraph, + features + ); + + assertThat(result.layers()) + .allSatisfy(layer -> assertThat(layer.weights()) + .noneMatch(weights -> TensorTestUtils.containsNaN(weights.data())) + ); + } + @ParameterizedTest @ValueSource(booleans = {false, true}) void trainsWithMeanAggregator(boolean useRelationshipWeight) { @@ -236,18 +288,7 @@ void testLosses() { assertThat(metrics.ranIterationsPerEpoch()).containsExactly(100, 100, 100, 100, 100, 100, 100, 100, 100, 100); assertThat(metrics.epochLosses().stream().mapToDouble(Double::doubleValue).toArray()) - .contains(new double[]{ - 18.25, - 16.31, - 16.41, - 16.21, - 14.96, - 14.97, - 14.31, - 16.17, - 14.90, - 15.58 - }, Offset.offset(0.05) + .contains(new double[]{19.55, 21.24, 19.90, 19.42, 17.87, 17.03, 17.04, 20.42, 15.86, 20.56}, Offset.offset(0.05) ); } @@ -280,18 +321,7 @@ void testLossesWithPoolAggregator() { assertThat(metrics.ranIterationsPerEpoch()).containsOnly(10); assertThat(metrics.epochLosses().stream().mapToDouble(Double::doubleValue).toArray()) - .contains(new double[]{ - 23.41, - 19.94, - 19.70, - 21.62, - 19.06, - 24.11, - 19.72, - 16.47, - 19.74, - 20.97 - }, Offset.offset(0.05) + .contains(new double[]{19.73, 21.25, 20.81, 23.13, 19.70, 25.34, 20.65, 17.10, 20.48, 21.51}, Offset.offset(0.05) ); } diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactoryTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactoryTest.java index 82ac56df6b1..b455e4f9512 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactoryTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/graphsage/algo/GraphSageTrainAlgorithmFactoryTest.java @@ -499,21 +499,21 @@ void testLogging() { "GraphSageTrain :: Train model :: Start", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Start", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 :: Start", - "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 :: Average loss per node: 26.49", + "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 :: Average loss per node: 25.58", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 100%", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 1 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 :: Start", - "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 :: Average loss per node: 25.58", + "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 :: Average loss per node: 26.69", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 100%", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Iteration 2 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 1 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Start", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 :: Start", - "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 :: Average loss per node: 25.28", + "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 :: Average loss per node: 23.29", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 100%", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 1 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 :: Start", - "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 :: Average loss per node: 25.23", + "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 :: Average loss per node: 22.88", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 100%", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Iteration 2 of 2 :: Finished", "GraphSageTrain :: Train model :: Epoch 2 of 2 :: Finished", diff --git a/ml/ml-core/src/main/java/org/neo4j/gds/ml/core/subgraph/SubGraph.java b/ml/ml-core/src/main/java/org/neo4j/gds/ml/core/subgraph/SubGraph.java index 3ad7285e082..362cc8c6710 100644 --- a/ml/ml-core/src/main/java/org/neo4j/gds/ml/core/subgraph/SubGraph.java +++ b/ml/ml-core/src/main/java/org/neo4j/gds/ml/core/subgraph/SubGraph.java @@ -69,24 +69,22 @@ public static List buildSubGraphs( } public static SubGraph buildSubGraph(long[] batchNodeIds, NeighborhoodFunction neighborhoodFunction, RelationshipWeights weightFunction) { - int[][] adjacency = new int[batchNodeIds.length][]; - int[] batchedNodeIds = new int[batchNodeIds.length]; + int[] mappedBatchNodeIds = new int[batchNodeIds.length]; // mapping original long-based nodeIds into consecutive int-based ids LocalIdMap idmap = new LocalIdMap(); // map the input node ids // this assures they are in consecutive order - for (long nodeId : batchNodeIds) { - idmap.toMapped(nodeId); - } - for (int nodeOffset = 0, nodeIdsLength = batchNodeIds.length; nodeOffset < nodeIdsLength; nodeOffset++) { - long nodeId = batchNodeIds[nodeOffset]; + int mappedNodeId = idmap.toMapped(batchNodeIds[nodeOffset]); + mappedBatchNodeIds[nodeOffset] = mappedNodeId; + } + int[][] adjacency = new int[idmap.size()][]; - batchedNodeIds[nodeOffset] = idmap.toMapped(nodeId); + for (int mappedNodeId = 0, mappedBatchIds = idmap.size(); mappedNodeId < mappedBatchIds; mappedNodeId++) { - var nodeNeighbors = neighborhoodFunction.sample(nodeId); + var nodeNeighbors = neighborhoodFunction.sample(idmap.toOriginal(mappedNodeId)); // map sampled neighbors into local id space // this also expands the id mapping as the neighbours could be not in the nodeIds[] @@ -94,10 +92,10 @@ public static SubGraph buildSubGraph(long[] batchNodeIds, NeighborhoodFunction n .mapToInt(idmap::toMapped) .toArray(); - adjacency[nodeOffset] = neighborInternalIds; + adjacency[mappedNodeId] = neighborInternalIds; } - return new SubGraph(adjacency, batchedNodeIds, idmap.originalIds(), weightFunction); + return new SubGraph(adjacency, mappedBatchNodeIds, idmap.originalIds(), weightFunction); } @Override diff --git a/ml/ml-core/src/test/java/org/neo4j/gds/ml/core/subgraph/SubGraphBuilderTest.java b/ml/ml-core/src/test/java/org/neo4j/gds/ml/core/subgraph/SubGraphBuilderTest.java index e290df76615..201d8143fc0 100644 --- a/ml/ml-core/src/test/java/org/neo4j/gds/ml/core/subgraph/SubGraphBuilderTest.java +++ b/ml/ml-core/src/test/java/org/neo4j/gds/ml/core/subgraph/SubGraphBuilderTest.java @@ -205,7 +205,7 @@ void shouldHandleDuplicatedNodes() { RelationshipWeights.UNWEIGHTED ); - assertEquals(6, subGraph.neighbors.length); + assertEquals(3, subGraph.neighbors.length); } @Test From 390af4a601aa4beecdee34d9a7b8be41bbb36b04 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Mon, 20 Mar 2023 15:09:59 +0100 Subject: [PATCH 286/400] kmeans array properties --- .../ROOT/partials/algorithms/kmeans/specific-configuration.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/partials/algorithms/kmeans/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/kmeans/specific-configuration.adoc index dc5d8c2a1f8..ac894342a41 100644 --- a/doc/modules/ROOT/partials/algorithms/kmeans/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/kmeans/specific-configuration.adoc @@ -1,4 +1,4 @@ -| nodeProperty | String | n/a | no | A node property to be used by the algorithm. +| nodeProperty | String | n/a | no | A node property corresponding to an array of floats used by K-Means to cluster nodes into communities. | k | Integer | 10 | yes | Number of desired clusters. | maxIterations | Integer | 10 | yes | The maximum number of iterations of K-Means to run. | deltaThreshold | Float | 0.05 | yes | Value as a percentage to determine when to stop early. If fewer than 'deltaThreshold * \|nodes\|' nodes change their cluster , the algorithm stops. Value must be between 0 (exclusive) and 1 (inclusive). From 94a6641af480d6a169cd8b7b544d0f23e2871432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 20 Mar 2023 15:53:34 +0100 Subject: [PATCH 287/400] Fix type definition --- .../partials/algorithms/page-rank/specific-configuration.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/partials/algorithms/page-rank/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/page-rank/specific-configuration.adoc index a86222bb207..3c502a76456 100644 --- a/doc/modules/ROOT/partials/algorithms/page-rank/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/page-rank/specific-configuration.adoc @@ -2,5 +2,5 @@ | xref:common-usage/running-algos.adoc#common-configuration-max-iterations[maxIterations] | Integer | 20 | yes | The maximum number of iterations of Page Rank to run. | xref:common-usage/running-algos.adoc#common-configuration-tolerance[tolerance] | Float | 0.0000001 | yes | Minimum change in scores between iterations. If all scores change less than the tolerance value the result is considered stable and the algorithm returns. | xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. -| sourceNodes | List or Node or Number | [] | yes | The nodes or node ids to use for computing Personalized Page Rank. +| sourceNodes | List of Node or Number | [] | yes | The nodes or node ids to use for computing Personalized Page Rank. | scaler | String | None | yes | The name of the scaler applied for the final scores. Supported values are `None`, `MinMax`, `Max`, `Mean`, `Log`, `L1Norm`, `L2Norm` and `StdScore`. From 5020b1a0122fda02d1a5960d4cc4fc557bff55e1 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Mon, 20 Mar 2023 15:59:43 +0000 Subject: [PATCH 288/400] Add relationshipWeightProperty config option to conductance and modularity metrics --- .../algorithms/alpha/conductance/specific-configuration.adoc | 1 + .../algorithms/alpha/modularity/specific-configuration.adoc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/modules/ROOT/partials/algorithms/alpha/conductance/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/alpha/conductance/specific-configuration.adoc index 9746bf81cf4..0f7807cb18d 100644 --- a/doc/modules/ROOT/partials/algorithms/alpha/conductance/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/alpha/conductance/specific-configuration.adoc @@ -1 +1,2 @@ +| xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. | communityProperty | String | n/a | no | The node property that holds the community ID as an integer for each node. Note that only non-negative community IDs are considered valid and will have their conductance computed. diff --git a/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc b/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc index 45968181212..958956007b8 100644 --- a/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc +++ b/doc/modules/ROOT/partials/algorithms/alpha/modularity/specific-configuration.adoc @@ -1,2 +1,2 @@ +| xref:common-usage/running-algos.adoc#common-configuration-relationship-weight-property[relationshipWeightProperty] | String | null | yes | Name of the relationship property to use as weights. If unspecified, the algorithm runs unweighted. | communityProperty | String | n/a | no | The node property that holds the community ID as an integer for each node. Note that only non-negative community IDs are considered valid and will have their modularity score computed. -| relationshipWeightProperty | String | null | yes | Relationship Weight. From 2d15195a967f7d31fd306a36b69bda70920e5ec2 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 17 Mar 2023 13:50:09 +0100 Subject: [PATCH 289/400] Update Neo4j Server installation instructions Fixes neo4j/graph-data-science#256 --- .../ROOT/pages/installation/neo4j-server.adoc | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index 745577e9a80..4a48981e189 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -1,14 +1,13 @@ [[neo4j-server]] = Neo4j Server -The GDS library is intended to be used on a standalone Neo4j server. +On a standalone Neo4j Server, GDS will need to be installed and configured manually. -On a standalone Neo4j Server, the library will need to be installed and configured manually. +1. Download `neo4j-graph-data-science-[version].zip` from the https://neo4j.com/download-center/#algorithms[Neo4j Download Center] -1. Download `neo4j-graph-data-science-[version].jar` from the https://neo4j.com/download-center/#algorithms[Neo4j Download Center] and copy it into the `$NEO4J_HOME/plugins` directory. +2. Unzip the archive and move the `neo4j-graph-data-science-[version].jar` file into the `$NEO4J_HOME/plugins` directory. - -2. Add the following to your `$NEO4J_HOME/conf/neo4j.conf` file: +3. Add the following to your `$NEO4J_HOME/conf/neo4j.conf` file: + ---- dbms.security.procedures.unrestricted=gds.* @@ -16,7 +15,7 @@ dbms.security.procedures.unrestricted=gds.* This configuration entry is necessary because the GDS library accesses low-level components of Neo4j to maximise performance. + -3. Check if the procedure allowlist is enabled in the `$NEO4J_HOME/conf/neo4j.conf` file and add the GDS library if necessary: +4. Check if the procedure allowlist is enabled in the `$NEO4J_HOME/conf/neo4j.conf` file and add the GDS library if necessary: + ---- dbms.security.procedures.allowlist=gds.* @@ -26,22 +25,22 @@ dbms.security.procedures.allowlist=gds.* NOTE: Before `Neo4j 4.2`, the configuration setting is called `dbms.security.procedures.whitelist` -4. Restart Neo4j +5. Restart Neo4j [[neo4j-server-verify]] == Verifying installation -To verify your installation, the library version can be printed by entering into the browser in Neo4j Desktop and calling the `gds.version()` function: +To verify your installation, print the version of Graph Data Science by opening Neo4j Browser and running the `gds.version()` function: [source, cypher, role=noplay] ---- -RETURN gds.version() +RETURN gds.version(); ---- -To list all installed algorithms, run the `gds.list()` procedure: +To list all available procedures, run the `gds.list()` procedure: [source, cypher, role=noplay] ---- -CALL gds.list() +CALL gds.list(); ---- From 3171cbf0522d5b563080d0051aa79e2a4c7bee82 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 17 Mar 2023 13:51:13 +0100 Subject: [PATCH 290/400] Remove mention of EOL Neo4j 4.2 --- doc/modules/ROOT/pages/algorithms/pregel-api.adoc | 2 -- doc/modules/ROOT/pages/installation/neo4j-desktop.adoc | 2 -- doc/modules/ROOT/pages/installation/neo4j-server.adoc | 3 --- 3 files changed, 7 deletions(-) diff --git a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc index c368009bf55..8aedc58a595 100644 --- a/doc/modules/ROOT/pages/algorithms/pregel-api.adoc +++ b/doc/modules/ROOT/pages/algorithms/pregel-api.adoc @@ -574,8 +574,6 @@ dbms.security.procedures.unrestricted=custom.pregel.proc.* dbms.security.procedures.allowlist=custom.pregel.proc.* ---- -NOTE: Before `Neo4j 4.2`, the configuration setting is called `dbms.security.procedures.whitelist` - [[algorithms-pregel-api-example]] == Examples diff --git a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc index fbbe1cc8ceb..21265eae603 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-desktop.adoc @@ -20,5 +20,3 @@ If the procedure allowlist is configured, make sure to also include procedures f ---- dbms.security.procedures.allowlist=gds.* ---- - -NOTE: Before `Neo4j 4.2`, the configuration setting is called `dbms.security.procedures.whitelist` diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index 4a48981e189..db27e65f13c 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -22,9 +22,6 @@ dbms.security.procedures.allowlist=gds.* ---- + -NOTE: Before `Neo4j 4.2`, the configuration setting is called `dbms.security.procedures.whitelist` - - 5. Restart Neo4j From c9a432a96ace4c8f7c5e0300598c286e391983e0 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 17 Mar 2023 14:01:13 +0100 Subject: [PATCH 291/400] Update link anchor --- doc/modules/ROOT/pages/installation/neo4j-server.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index db27e65f13c..c449b1676ef 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -3,7 +3,7 @@ On a standalone Neo4j Server, GDS will need to be installed and configured manually. -1. Download `neo4j-graph-data-science-[version].zip` from the https://neo4j.com/download-center/#algorithms[Neo4j Download Center] +1. Download `neo4j-graph-data-science-[version].zip` from the https://neo4j.com/download-center/#graph-data-science[Neo4j Download Center] 2. Unzip the archive and move the `neo4j-graph-data-science-[version].jar` file into the `$NEO4J_HOME/plugins` directory. From d535310f58ee3943fd4f90c9bb018d38d38fbcd4 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Tue, 21 Mar 2023 14:22:55 +0100 Subject: [PATCH 292/400] Fix link anchor This one works great --- doc/modules/ROOT/pages/installation/neo4j-server.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/installation/neo4j-server.adoc b/doc/modules/ROOT/pages/installation/neo4j-server.adoc index c449b1676ef..fcc0af6aec3 100644 --- a/doc/modules/ROOT/pages/installation/neo4j-server.adoc +++ b/doc/modules/ROOT/pages/installation/neo4j-server.adoc @@ -3,7 +3,7 @@ On a standalone Neo4j Server, GDS will need to be installed and configured manually. -1. Download `neo4j-graph-data-science-[version].zip` from the https://neo4j.com/download-center/#graph-data-science[Neo4j Download Center] +1. Download `neo4j-graph-data-science-[version].zip` from the https://neo4j.com/download-center/#ngds[Neo4j Download Center] 2. Unzip the archive and move the `neo4j-graph-data-science-[version].jar` file into the `$NEO4J_HOME/plugins` directory. From 9ce101c6da7a4f1124b9a9aebd221ac2eecdefdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 22 Mar 2023 16:43:33 +0100 Subject: [PATCH 293/400] Document create db through arrow is not supported in Aura --- doc/modules/ROOT/pages/graph-project-apache-arrow.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc index 6db24ce6a99..8975dddd948 100644 --- a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc +++ b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc @@ -235,6 +235,8 @@ The timeout can be configured via the `gds.arrow.abortion_timeout` setting, for == Creating a Neo4j database +include::partial$/common-usage/not-on-aurads-note.adoc[] + The xref:graph-project-apache-arrow.adoc#arrow-client-server-protocol[Client-Server protocol] can also be used to create a new Neo4j database instead of an in-memory graph. To initialize a database import process, we need to change the initial action type to `CREATE_DATABASE`. The action body is a JSON document containing the configuration for the import process: From 678dcb37db3b2dfb73efdf34e033a4699c490fba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 23 Mar 2023 11:55:46 +0100 Subject: [PATCH 294/400] Publish 5.6 compat projects Co-Authored-By: Mats Rydberg --- build.gradle | 2 ++ settings.gradle | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/build.gradle b/build.gradle index 703608b70c2..892e9123e2d 100644 --- a/build.gradle +++ b/build.gradle @@ -34,6 +34,7 @@ ext { project(':neo4j-kernel-adapter-5.3'), project(':neo4j-kernel-adapter-5.4'), project(':neo4j-kernel-adapter-5.5'), + project(':neo4j-kernel-adapter-5.6'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -42,6 +43,7 @@ ext { project(':storage-engine-adapter-5.3'), project(':storage-engine-adapter-5.4'), project(':storage-engine-adapter-5.5'), + project(':storage-engine-adapter-5.6'), ] ] } diff --git a/settings.gradle b/settings.gradle index 6027407577a..7786a5fbd66 100644 --- a/settings.gradle +++ b/settings.gradle @@ -139,6 +139,9 @@ project(':neo4j-kernel-adapter-5.4').projectDir = file('compatibility/5.4/neo4j- include('neo4j-kernel-adapter-5.5') project(':neo4j-kernel-adapter-5.5').projectDir = file('compatibility/5.5/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.6') +project(':neo4j-kernel-adapter-5.6').projectDir = file('compatibility/5.6/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -226,6 +229,9 @@ project(':storage-engine-adapter-5.4').projectDir = file('compatibility/5.4/stor include('storage-engine-adapter-5.5') project(':storage-engine-adapter-5.5').projectDir = file('compatibility/5.5/storage-engine-adapter') +include('storage-engine-adapter-5.6') +project(':storage-engine-adapter-5.6').projectDir = file('compatibility/5.6/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') From 9a2d13f485b0d2cbfe9861bfdea9ca24d9547c21 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 23 Mar 2023 12:34:50 +0100 Subject: [PATCH 295/400] Add fake impl of SEFactory to 5.6 layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- .../_56/InMemoryStorageEngineFactory.java | 268 ++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java b/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..fc6e82cb270 --- /dev/null +++ b/compatibility/5.6/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._56; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported56"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.6 storage engine requires JDK17"); + } +} From ca061d50c07d29f46e8627e6ff4a9c6512a1defb Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 23 Mar 2023 14:10:19 +0100 Subject: [PATCH 296/400] Add kernel dependency and missing StorageEngine class for RC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- compatibility/5.6/storage-engine-adapter/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/compatibility/5.6/storage-engine-adapter/build.gradle b/compatibility/5.6/storage-engine-adapter/build.gradle index 0f28b9de440..e7ca6e2ff23 100644 --- a/compatibility/5.6/storage-engine-adapter/build.gradle +++ b/compatibility/5.6/storage-engine-adapter/build.gradle @@ -44,6 +44,7 @@ if (ver.'neo4j'.startsWith('5.')) { dependencies { annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' implementation project(':neo4j-adapter') implementation project(':storage-engine-adapter-api') From d4346dc33dcc2e5578723165f9b3891fe3d3d638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 23 Mar 2023 14:52:47 +0100 Subject: [PATCH 297/400] Clarify relationship orientation in cypher aggregation --- .../graph-project-cypher-aggregation.adoc | 62 +++++++++++++++++-- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc index dbfffe41287..9248d41654f 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc @@ -265,13 +265,65 @@ The value for `relationshipType` must be a `String`: === Relationship orientation The native projection supports specifying an orientation per relationship type. -The Cypher Aggregation will treat every relationship returned by the relationship query as if it was in `NATURAL` orientation. -It is thus not possible to project graphs in `UNDIRECTED` or `REVERSE` orientation when Cypher projections are used. +The Cypher Aggregation will treat every relationship returned by the relationship query as if it was in `NATURAL` orientation by default. -[NOTE] +==== Reverse relationships + +Achieving the reverse orientation can be achieved by simply switched the source and target nodes. + +[role=query-example] +-- +.Project `Person` and `Book` nodes and `KNOWS` and `READ` relationships: +[source, cypher, role=noplay] +---- +MATCH (source)-[r:KNOWS|READ]->(target) +WHERE source:Book OR source:Person +WITH gds.alpha.graph.project( + 'graphWithReverseRelationships', + target, + source, + {}, + {} +) as g +RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels +---- + +.Results +[opts="header", cols="1,1,1"] +|=== +| graph | nodes | rels +| "graphWithReverseRelationships" | 5 | 6 +|=== -- -Some algorithms require that the graph was loaded with `UNDIRECTED` orientation. -These algorithms can not be used with a graph projected by a Cypher Aggregation. + +==== Undirected relationships + +To project relationships as undirected, you can specify the `undirectedRelationshipTypes` parameter. + +[role=query-example] +-- +.Project `Person` and `Book` nodes and `KNOWS` and `READ` relationships: +[source, cypher, role=noplay] +---- +MATCH (source)-[r:KNOWS|READ]->(target) +WHERE source:Book OR source:Person +WITH gds.alpha.graph.project( + 'graphWithUndirectedRelationships', + source, + target, + {}, + {}, + {undirectedRelationshipTypes: ['*']} +) as g +RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels +---- + +.Results +[opts="header", cols="1,1,1"] +|=== +| graph | nodes | rels +| "graphWithUndirectedRelationships" | 5 | 12 +|=== -- From 907bf6106e71fea238535ae3e93f95fb57f029ef Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 23 Mar 2023 16:42:59 +0100 Subject: [PATCH 298/400] Bump to next AuraDS version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 65ecbe32df7..a43e93d2470 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.2' - gdsAuraVersion = '15' + gdsAuraVersion = '16' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 25d1f86fd36ccf5975c9c6841e0d6134bbabed6d Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 23 Mar 2023 16:51:06 +0100 Subject: [PATCH 299/400] Bump to next patch version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jonatan Jäderberg --- README.adoc | 2 +- doc/modules/ROOT/pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.adoc b/README.adoc index 64d82e539e1..3126b5909e4 100644 --- a/README.adoc +++ b/README.adoc @@ -77,7 +77,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library .6+<.^|GDS 2.2.x |Neo4j 4.3.15 - 4.3.23 -.12+.^|Java 11 & Java 17 +.13+.^|Java 11 & Java 17 |Neo4j 4.4.9 - 4.4.16 |Neo4j 5.1.0 |Neo4j 5.2.0 diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 5a7a4c839a7..79ea3867dcd 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.2" +| "2.3.3" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index 85af697057e..ae83fd656d5 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.2' + gdsVersion = '2.3.3' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index a43e93d2470..7aa39f5d851 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.3.2' + gdsBaseVersion = '2.3.3' gdsAuraVersion = '16' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From 26ce707a7adedd0c6276c74ef2c49752bc2c74ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 23 Mar 2023 14:38:54 +0100 Subject: [PATCH 300/400] Fix outdated docs around undirected rels in APSP doc --- .../neo4j/gds/AllShortestPathsDocTest.java | 160 ------------------ .../gds/doc/AllPairsShortestPathDocTest.java | 50 ++++++ .../all-pairs-shortest-path.adoc | 84 ++++----- 3 files changed, 95 insertions(+), 199 deletions(-) delete mode 100644 alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java create mode 100644 doc-test/src/test/java/org/neo4j/gds/doc/AllPairsShortestPathDocTest.java diff --git a/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java b/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java deleted file mode 100644 index 197a08375ad..00000000000 --- a/alpha/alpha-proc/src/test/java/org/neo4j/gds/AllShortestPathsDocTest.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.neo4j.gds.catalog.GraphProjectProc; -import org.neo4j.gds.functions.IsFiniteFunc; -import org.neo4j.gds.shortestpaths.AllShortestPathsProc; -import org.neo4j.graphdb.Result; - -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class AllShortestPathsDocTest extends BaseProcTest { - private static final String NL = System.lineSeparator(); - public static final String DB_CYPHER = "CREATE " + - " (a:Loc {name:'A'})," + - " (b:Loc {name:'B'})," + - " (c:Loc {name:'C'})," + - " (d:Loc {name:'D'})," + - " (e:Loc {name:'E'})," + - " (f:Loc {name:'F'})," + - " (a)-[:ROAD {cost:50}]->(b)," + - " (a)-[:ROAD {cost:50}]->(c)," + - " (a)-[:ROAD {cost:100}]->(d)," + - " (b)-[:ROAD {cost:40}]->(d)," + - " (c)-[:ROAD {cost:40}]->(d)," + - " (c)-[:ROAD {cost:80}]->(e)," + - " (d)-[:ROAD {cost:30}]->(e)," + - " (d)-[:ROAD {cost:80}]->(f)," + - " (e)-[:ROAD {cost:40}]->(f);"; - - @BeforeEach - void setupGraph() throws Exception { - registerProcedures(AllShortestPathsProc.class, GraphProjectProc.class); - registerFunctions(IsFiniteFunc.class); - runQuery(DB_CYPHER); - } - - @Test - void shouldStream() { - var createQuery = "CALL gds.graph.project(" + - " 'nativeGraph', " + - " 'Loc', " + - " {" + - " ROAD: {" + - " type: 'ROAD'," + - " properties: 'cost'" + - " }" + - " }" + - ")" + - "YIELD graphName"; - runQuery(createQuery); - String query = " CALL gds.alpha.allShortestPaths.stream('nativeGraph', {" + - " relationshipWeightProperty: 'cost'" + - " })" + - " YIELD sourceNodeId, targetNodeId, distance" + - " WITH sourceNodeId, targetNodeId, distance" + - " WHERE gds.util.isFinite(distance) = true" + - - " MATCH (source:Loc) WHERE id(source) = sourceNodeId" + - " MATCH (target:Loc) WHERE id(target) = targetNodeId" + - " WITH source, target, distance WHERE source <> target" + - - " RETURN source.name AS source, target.name AS target, distance" + - " ORDER BY distance DESC, source ASC, target ASC" + - " LIMIT 10"; - - String actual = runQuery(query, AllShortestPathsDocTest::resultAsStringNoDeprecation); - String expected = "+----------------------------+" + NL + - "| source | target | distance |" + NL + - "+----------------------------+" + NL + - "| \"A\" | \"F\" | 160.0 |" + NL + - "| \"A\" | \"E\" | 120.0 |" + NL + - "| \"B\" | \"F\" | 110.0 |" + NL + - "| \"C\" | \"F\" | 110.0 |" + NL + - "| \"A\" | \"D\" | 90.0 |" + NL + - "| \"B\" | \"E\" | 70.0 |" + NL + - "| \"C\" | \"E\" | 70.0 |" + NL + - "| \"D\" | \"F\" | 70.0 |" + NL + - "| \"A\" | \"B\" | 50.0 |" + NL + - "| \"A\" | \"C\" | 50.0 |" + NL + - "+----------------------------+" + NL + - "10 rows" + NL; - - assertEquals(expected, actual); - } - - @Test - void shouldStreamWithCypherProjection() { - var createQuery = " CALL gds.graph.project.cypher(" + - " 'cypherGraph'," + - " 'MATCH (n:Loc) RETURN id(n) AS id', " + - " 'MATCH (n:Loc)-[r:ROAD]-(p:Loc) RETURN id(n) AS source, id(p) AS target, r.cost AS cost'" + - " ) " + - " YIELD graphName"; - runQuery(createQuery); - String query = " CALL gds.alpha.allShortestPaths.stream('cypherGraph', {" + - " relationshipWeightProperty: 'cost'" + - " })" + - " YIELD sourceNodeId, targetNodeId, distance" + - " WITH sourceNodeId, targetNodeId, distance" + - " WHERE gds.util.isFinite(distance) = true" + - - " MATCH (source:Loc) WHERE id(source) = sourceNodeId" + - " MATCH (target:Loc) WHERE id(target) = targetNodeId" + - " WITH source, target, distance WHERE source <> target" + - - " RETURN source.name AS source, target.name AS target, distance" + - " ORDER BY distance DESC, source ASC, target ASC" + - " LIMIT 10"; - - String actual = runQuery(query, AllShortestPathsDocTest::resultAsStringNoDeprecation); - String expected = "+----------------------------+" + NL + - "| source | target | distance |" + NL + - "+----------------------------+" + NL + - "| \"A\" | \"F\" | 160.0 |" + NL + - "| \"F\" | \"A\" | 160.0 |" + NL + - "| \"A\" | \"E\" | 120.0 |" + NL + - "| \"E\" | \"A\" | 120.0 |" + NL + - "| \"B\" | \"F\" | 110.0 |" + NL + - "| \"C\" | \"F\" | 110.0 |" + NL + - "| \"F\" | \"B\" | 110.0 |" + NL + - "| \"F\" | \"C\" | 110.0 |" + NL + - "| \"A\" | \"D\" | 90.0 |" + NL + - "| \"D\" | \"A\" | 90.0 |" + NL + - "+----------------------------+" + NL + - "10 rows" + NL; - - assertEquals(expected, actual); - } - - private static final Pattern DEPRECATION_NOTE = Pattern.compile( - "The query used a deprecated function.+", - Pattern.DOTALL - ); - - private static String resultAsStringNoDeprecation(Result result) { - return DEPRECATION_NOTE.matcher(result.resultAsString()).replaceAll(""); - } -} diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/AllPairsShortestPathDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/AllPairsShortestPathDocTest.java new file mode 100644 index 00000000000..21dbcb12a0d --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/AllPairsShortestPathDocTest.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc; + +import org.neo4j.gds.catalog.GraphProjectProc; +import org.neo4j.gds.functions.AsNodeFunc; +import org.neo4j.gds.functions.IsFiniteFunc; +import org.neo4j.gds.projection.CypherAggregation; +import org.neo4j.gds.shortestpaths.AllShortestPathsProc; + +import java.util.List; + +class AllPairsShortestPathDocTest extends SingleFileDocTestBase { + + @Override + protected List> procedures() { + return List.of( + GraphProjectProc.class, + CypherAggregation.class, + AllShortestPathsProc.class + ); + } + + @Override + protected List> functions() { + return List.of(AsNodeFunc.class, IsFiniteFunc.class); + } + + @Override + protected String adocFile() { + return "pages/alpha-algorithms/all-pairs-shortest-path.adoc"; + } +} diff --git a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc index 72e7f6f27fb..00f98c46673 100644 --- a/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc +++ b/doc/modules/ROOT/pages/alpha-algorithms/all-pairs-shortest-path.adoc @@ -65,7 +65,7 @@ YIELD startNodeId, targetNodeId, distance image::example-graphs/shortest-path_graph.png[] .The following will create a sample graph: -[source, cypher, role=noplay] +[source, cypher, role=noplay setup-query] ---- CREATE (a:Loc {name: 'A'}), (b:Loc {name: 'B'}), @@ -85,8 +85,10 @@ CREATE (a:Loc {name: 'A'}), ---- +=== Using native projection + .The following will project and store a graph using native projection: -[source, cypher, role=noplay] +[source, cypher, role=noplay graph-project-query, group=native] ---- CALL gds.graph.project( 'nativeGraph', @@ -100,8 +102,10 @@ CALL gds.graph.project( YIELD graphName ---- +[role=query-example] +-- .The following will run the algorithm and stream results: -[source, cypher, role=noplay] +[source, cypher, role=noplay, group=native] ---- CALL gds.alpha.allShortestPaths.stream('nativeGraph', { relationshipWeightProperty: 'cost' @@ -122,42 +126,43 @@ LIMIT 10 .Results [opts="header",cols="1,1,1"] |=== -| Source | Target | Cost -| A | F | 160 -| A | E | 120 -| B | F | 110 -| C | F | 110 -| A | D | 90 -| B | E | 70 -| C | E | 70 -| D | F | 70 -| A | B | 50 -| A | C | 50 +| source | target | distance +| "A" | "F" | 160 +| "A" | "E" | 120 +| "B" | "F" | 110 +| "C" | "F" | 110 +| "A" | "D" | 90 +| "B" | "E" | 70 +| "C" | "E" | 70 +| "D" | "F" | 70 +| "A" | "B" | 50 +| "A" | "C" | 50 |=== +-- This query returned the top 10 pairs of nodes that are the furthest away from each other. F and E appear to be quite distant from the others. -For now, only single-source shortest path support loading the relationship as undirected, but we can use Cypher loading to help us solve this. -Undirected graph can be represented as https://en.wikipedia.org/wiki/Bidirected_graph[Bidirected graph], which is a directed graph in which the reverse of every relationship is also a relationship. - -We do not have to save this reversed relationship, we can project it using *Cypher loading*. -Note that relationship query does not specify direction of the relationship. -This is applicable to all other algorithms that use Cypher loading. +=== Using Cypher aggregation -.The following will project and store an undirected graph using cypher projection: -[source, cypher, role=noplay] +.The following will project and store an undirected graph using cypher aggregation: +[source, cypher, role=noplay graph-project-query, group=cypher] ---- -CALL gds.graph.project.cypher( +MATCH (src:Loc)-[r:ROAD]->(trg:Loc) +RETURN gds.alpha.graph.project( 'cypherGraph', - 'MATCH (n:Loc) RETURN id(n) AS id', - 'MATCH (n:Loc)-[r:ROAD]-(p:Loc) RETURN id(n) AS source, id(p) AS target, r.cost AS cost' -) -YIELD graphName + src, + trg, + {}, + {relationshipType: type(r), properties: {cost: r.cost}}, + {undirectedRelationshipTypes: ['ROAD'] +}) ---- +[role=query-example] +-- .The following will run the algorithm, treating the graph as undirected: -[source, cypher, role=noplay] +[source, cypher, role=noplay, group=cypher] ---- CALL gds.alpha.allShortestPaths.stream('cypherGraph', { relationshipWeightProperty: 'cost' @@ -178,15 +183,16 @@ LIMIT 10 .Results [opts="header",cols="1,1,1"] |=== -| Source | Target | Cost -| A | F | 160 -| F | A | 160 -| A | E | 120 -| E | A | 120 -| B | F | 110 -| C | F | 110 -| F | B | 110 -| F | C | 110 -| A | D | 90 -| D | A | 90 +| source | target | distance +| "A" | "F" | 160 +| "F" | "A" | 160 +| "A" | "E" | 120 +| "E" | "A" | 120 +| "B" | "F" | 110 +| "C" | "F" | 110 +| "F" | "B" | 110 +| "F" | "C" | 110 +| "A" | "D" | 90 +| "D" | "A" | 90 |=== +-- From 998a903edb16a5eb0e984a9253b6abd52a180eab Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 30 Mar 2023 10:39:15 +0100 Subject: [PATCH 301/400] Update gradle to use Neo4j 4.4.19 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index aba1af8db72..e9c6ab5ee99 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.18'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.19'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), From 94706f71b8d42805ff2d976f483316316c3a4e6f Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 30 Mar 2023 10:50:12 +0100 Subject: [PATCH 302/400] Update README to reflect Neo4j 4.4.19 support --- README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index 3126b5909e4..7906d6dc416 100644 --- a/README.adoc +++ b/README.adoc @@ -84,7 +84,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.3.0 |Neo4j 5.4.0 .7+<.^|GDS 2.3.x -|Neo4j 4.4.9 - 4.4.18 +|Neo4j 4.4.9 - 4.4.19 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 From f20f47e4ad3c18248933d7b4327ce8554901d1e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 31 Mar 2023 14:42:11 +0200 Subject: [PATCH 303/400] Allow backups to skip graphs --- doc/modules/ROOT/pages/management-ops/backup-restore.adoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc index 263a7fcc9f4..56afdf9793d 100644 --- a/doc/modules/ROOT/pages/management-ops/backup-restore.adoc +++ b/doc/modules/ROOT/pages/management-ops/backup-restore.adoc @@ -44,8 +44,9 @@ YIELD .Configuration [opts="header",cols="1,1,1,4"] |=== -| Name | Type | Default | Description -| concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. +| Name | Type | Default | Description +| concurrency | Integer | 4 | The number of concurrent threads used for performing the backup. +| includeGraphs | Boolean | true | Flag to decide whether only models or also graphs should be backed up. |=== .Results From ecd8642f95d4315702449f657305a443ea69171e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 3 Apr 2023 13:44:07 +0200 Subject: [PATCH 304/400] Improve wording Co-authored-by: Nicola Vitucci --- .../graph-project-cypher-aggregation.adoc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc index 9248d41654f..254c3f1ab16 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc @@ -228,7 +228,7 @@ WITH gds.alpha.graph.project( relationshipType: type(r) } ) AS g -RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels ---- .Results @@ -269,7 +269,7 @@ The Cypher Aggregation will treat every relationship returned by the relationshi ==== Reverse relationships -Achieving the reverse orientation can be achieved by simply switched the source and target nodes. +The orientation of a relationship can be reversed by switching the source and target nodes. [role=query-example] -- @@ -285,7 +285,7 @@ WITH gds.alpha.graph.project( {}, {} ) as g -RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels ---- .Results @@ -298,7 +298,7 @@ RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels ==== Undirected relationships -To project relationships as undirected, you can specify the `undirectedRelationshipTypes` parameter. +Relationships can be projected as undirected by specifying the `undirectedRelationshipTypes` parameter. [role=query-example] -- @@ -315,7 +315,7 @@ WITH gds.alpha.graph.project( {}, {undirectedRelationshipTypes: ['*']} ) as g -RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels ---- .Results @@ -352,7 +352,7 @@ WITH gds.alpha.graph.project( targetNodeProperties: target { age: coalesce(target.age, 18), price: coalesce(target.price, 5.0), .ratings } } ) as g -RETURN g.graphName AS graph , g.nodeCount AS nodes, g.relationshipCount AS rels +RETURN g.graphName AS graph, g.nodeCount AS nodes, g.relationshipCount AS rels ---- .Results From 1e675de7cb39aa84fc22264d80d8ee390f3622d6 Mon Sep 17 00:00:00 2001 From: lidiazuin Date: Mon, 3 Apr 2023 14:33:18 +0200 Subject: [PATCH 305/400] Updating the link to this paper as the previous link was leading to 404. This was reported on SEMRush --- doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc b/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc index d89d2a2dd56..f475501c9cd 100644 --- a/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc +++ b/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc @@ -30,7 +30,7 @@ The implementation requires _O(n + m)_ space and runs in _O(n * m)_ time, where For more information on this algorithm, see: -* https://www.eecs.wsu.edu/~assefaw/CptS580-06/papers/brandes01centrality.pdf[A Faster Algorithm for Betweenness Centrality^] +* http://snap.stanford.edu/class/cs224w-readings/brandes01centrality.pdf[A Faster Algorithm for Betweenness Centrality^] * https://www.uni-konstanz.de/mmsp/pubsys/publishedFiles/BrPi07.pdf[Centrality Estimation in Large Networks^] * http://moreno.ss.uci.edu/23.pdf[A Set of Measures of Centrality Based on Betweenness^] From ec11e9ba410b706dd261c6f40ac9c37f8f216e3c Mon Sep 17 00:00:00 2001 From: lidiazuin Date: Mon, 3 Apr 2023 14:35:51 +0200 Subject: [PATCH 306/400] updating link from http to https --- doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc b/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc index f475501c9cd..8702d3d3859 100644 --- a/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc +++ b/doc/modules/ROOT/pages/algorithms/betweenness-centrality.adoc @@ -30,7 +30,7 @@ The implementation requires _O(n + m)_ space and runs in _O(n * m)_ time, where For more information on this algorithm, see: -* http://snap.stanford.edu/class/cs224w-readings/brandes01centrality.pdf[A Faster Algorithm for Betweenness Centrality^] +* https://snap.stanford.edu/class/cs224w-readings/brandes01centrality.pdf[A Faster Algorithm for Betweenness Centrality^] * https://www.uni-konstanz.de/mmsp/pubsys/publishedFiles/BrPi07.pdf[Centrality Estimation in Large Networks^] * http://moreno.ss.uci.edu/23.pdf[A Set of Measures of Centrality Based on Betweenness^] From 3497a7c4ceb72af7a89bf914cba91c0b6ca64ee9 Mon Sep 17 00:00:00 2001 From: lidiazuin Date: Mon, 3 Apr 2023 14:41:09 +0200 Subject: [PATCH 307/400] another external broken link for a paper --- .../ROOT/pages/machine-learning/node-embeddings/fastrp.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc index fb46b397873..ae7f6bee0fe 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/fastrp.adoc @@ -65,7 +65,7 @@ In general, it is recommended to first use `UNDIRECTED` as this is what the orig For more information on this algorithm see: * https://arxiv.org/pdf/1908.11512.pdf[H. Chen, S.F. Sultan, Y. Tian, M. Chen, S. Skiena: Fast and Accurate Network Embeddings via Very Sparse Random Projection, 2019.^] -* https://core.ac.uk/download/pdf/82724427.pdf[Dimitris Achlioptas. Database-friendly random projections: Johnson-Lindenstrauss with binary coins. Journal of Computer and System Sciences, 66(4):671–687, 2003.] +* https://www.sciencedirect.com/science/article/pii/S0022000003000254[Dimitris Achlioptas. Database-friendly random projections: Johnson-Lindenstrauss with binary coins. Journal of Computer and System Sciences, 66(4):671–687, 2003.] === Node properties From 6a35ae0d6703212ebf919816e61a4d8e27a426f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 3 Apr 2023 11:50:25 +0200 Subject: [PATCH 308/400] Add getHighId to Neo4jApiProxy Adapts compat after neo-technology/neo4j#20072 --- .../main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java | 5 +++++ .../main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java | 5 +++++ .../main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java | 5 +++++ .../main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java | 5 +++++ .../main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java | 5 +++++ .../main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java | 5 +++++ .../main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java | 5 +++++ .../src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java | 2 ++ .../src/main/java/org/neo4j/gds/compat/Neo4jProxy.java | 4 ++++ .../gds/core/io/db/ProgressTrackerExecutionMonitor.java | 4 ++-- 10 files changed, 43 insertions(+), 2 deletions(-) diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index 73ffa9abab5..c6b32cbccd1 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -187,6 +187,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index dd9564803aa..83a1ac02da9 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -201,6 +201,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index f7c51bffc1a..d8ed7eae7a7 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -201,6 +201,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index f4b6072fe0a..4902182be1e 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -201,6 +201,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index 8f97bac8b98..32662efb8fc 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -201,6 +201,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java index 16a57839d6c..be8b09f7925 100644 --- a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -202,6 +202,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java index 125ce3a89a9..bd4ba3f4682 100644 --- a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -202,6 +202,11 @@ public long getHighestPossibleIdInUse( return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); } + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getHighId(); + } + @Override public List> entityCursorScan( KernelTransaction transaction, diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index ed0fedb0173..4cddd86ccea 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -112,6 +112,8 @@ long getHighestPossibleIdInUse( KernelTransaction kernelTransaction ); + long getHighId(RecordStore recordStore); + List> entityCursorScan( KernelTransaction transaction, int[] labelIds, diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index c319f814b1c..f081d37c75b 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -131,6 +131,10 @@ public static long getHighestPossibleIdInUse( return IMPL.getHighestPossibleIdInUse(recordStore, kernelTransaction); } + public static long getHighId(RecordStore recordStore) { + return IMPL.getHighId(recordStore); + }; + public static List> entityCursorScan( KernelTransaction transaction, int[] labelIds, diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java index 728a238b00e..56fbfccccf8 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/db/ProgressTrackerExecutionMonitor.java @@ -114,8 +114,8 @@ public void start(StageExecution execution) { this.stageProgressCurrent.set(0); var neoStores = this.dependencyResolver.resolveDependency(BatchingNeoStores.class); - var relationshipRecordIdCount = neoStores.getRelationshipStore().getHighId(); - var groupCount = neoStores.getTemporaryRelationshipGroupStore().getHighId(); + var relationshipRecordIdCount = Neo4jProxy.getHighId(neoStores.getRelationshipStore()); + var groupCount = Neo4jProxy.getHighId(neoStores.getTemporaryRelationshipGroupStore()); switch (execution.getStageName()) { case NodeDegreeCountStage.NAME: From bb3193ee29eed6428f32cc8bba1fa80f7280844d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 5 Apr 2023 09:54:13 +0200 Subject: [PATCH 309/400] Add log4j-api dependency Otherwise, we get error: cannot access ExtendedLoggerWrapper `return new Log4jLogProvider(outputStream, level.orElse(Level.INFO)).getLog(category.orElse(""));`. --- compatibility/api/neo4j-kernel-adapter/build.gradle | 2 ++ gradle/dependencies.gradle | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/compatibility/api/neo4j-kernel-adapter/build.gradle b/compatibility/api/neo4j-kernel-adapter/build.gradle index db3367534e7..943fb9ef50e 100644 --- a/compatibility/api/neo4j-kernel-adapter/build.gradle +++ b/compatibility/api/neo4j-kernel-adapter/build.gradle @@ -13,6 +13,8 @@ dependencies { compileOnly group: 'org.immutables', name: 'builder', version: ver.'immutables' compileOnly group: 'org.jetbrains', name: 'annotations', version: ver.'jetbrains-annotations' compileOnly group: 'org.eclipse.collections', name: 'eclipse-collections', version: ver.'eclipse-collections' + compileOnly group: 'org.apache.logging.log4j', name: 'log4j-api', version: ver.'log4j' + neodeps().each { compileOnly(group: 'org.neo4j', name: it, version: ver.'neo4j') { transitive = false diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index e9c6ab5ee99..b693a0d6bd1 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -22,6 +22,18 @@ ext { '5.4': '2.13.8', '5.5': '2.13.8', '5.6': '2.13.8', + '5.7': '2.13.10' + ] + + log4js = [ + '4.4': '2.17.0', + '5.1': '2.18.0', + '5.2': '2.19.0', + '5.3': '2.19.0', + '5.4': '2.19.0', + '5.5': '2.19.0', + '5.6': '2.19.0', + '5.7': '2.20.0' ] ver = [ @@ -58,6 +70,7 @@ ext { 'jqwik' : '1.6.5', 'junit4' : '4.13.2', 'junit5bom' : '5.9.1', + 'log4j' : log4js[neo4j_minor], 'memoryfilesystem' : '2.2.0', 'mockito' : '4.11.0', 'mockito-junit-jupiter': '4.11.0', From 511a1b755b334bcfa7d70bc4406df72e44f46be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 5 Apr 2023 10:20:59 +0200 Subject: [PATCH 310/400] Adapt to removed pageCacheMemory method https://github.com/neo-technology/neo4j/commit/c89def41c30742f787c6c903d7768f3ad5c4c342 --- .../gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java b/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java index d0f132f6dad..fe2ee835695 100644 --- a/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java +++ b/io/core/src/test/java/org/neo4j/gds/core/io/db/GraphStoreToDatabaseExporterConfigTest.java @@ -40,7 +40,6 @@ void testToBatchImporterConfig() { var pbiConfig = config.toBatchImporterConfig(); assertThat(pbiConfig.batchSize()).isEqualTo(1337); - assertThat(pbiConfig.pageCacheMemory()).isEqualTo(100_000L); assertThat(pbiConfig.highIO()).isTrue(); assertThat(Neo4jProxy.writeConcurrency(pbiConfig)).isEqualTo(42); assertThat(pbiConfig.indexConfig().createLabelIndex()).isTrue(); From 377544f95fa45eb30eb0f0d1f86f433d4f846e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 24 Mar 2023 09:50:29 +0100 Subject: [PATCH 311/400] End subtaskWithFailure in failure case --- .../gds/core/io/file/FileToGraphStoreImporter.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java index 8ff376380ce..1bda4e211c0 100644 --- a/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java +++ b/io/core/src/main/java/org/neo4j/gds/core/io/file/FileToGraphStoreImporter.java @@ -97,13 +97,19 @@ protected FileToGraphStoreImporter( public UserGraphStore run() { var fileInput = fileInput(importPath); this.progressTracker = createProgressTracker(fileInput); - progressTracker.beginSubTask(); + try { + progressTracker.beginSubTask(); importGraphStore(fileInput); graphStoreBuilder.schema(graphSchemaBuilder.build()); - return ImmutableUserGraphStore.of(fileInput.userName(), graphStoreBuilder.build()); - } finally { + var userGraphStore = ImmutableUserGraphStore.of(fileInput.userName(), graphStoreBuilder.build()); progressTracker.endSubTask(); + + return userGraphStore; + } catch (Exception e) { + progressTracker.endSubTaskWithFailure(); + + throw e; } } From 4304c30c112f2a1e0bc92ea2bb4122703cc5d66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 5 Apr 2023 11:05:01 +0200 Subject: [PATCH 312/400] Add renamed transaction_max_size to debug sysInfo Co-authored-by: Martin Junghanns --- proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java | 1 + 1 file changed, 1 insertion(+) diff --git a/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java b/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java index a8c0c6f1e4d..c5316055b87 100644 --- a/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java +++ b/proc/sysinfo/src/main/java/org/neo4j/gds/SysInfoProc.java @@ -309,6 +309,7 @@ private static void configInfo(Config config, Consumer builder) { trySetting("dbms.tx_state.max_off_heap_memory", config, builder); trySetting("dbms.memory.off_heap.max_size", config, builder); trySetting("server.memory.off_heap.max_size", config, builder); + trySetting("server.memory.off_heap.transaction_max_size", config, builder); trySetting("dbms.memory.transaction.global_max_size", config, builder); trySetting("dbms.memory.transaction.total.max", config, builder); From c8aa8199c98ef7993cb7c97f1ba5efdeb5ce6000 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 6 Apr 2023 09:12:37 +0200 Subject: [PATCH 313/400] Fix typo in supported neo4j versions --- .../ROOT/pages/installation/supported-neo4j-versions.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index f0c38f388c3..2e567fb174c 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,7 +10,7 @@ Time to upgrade! [opts=header] |=== | Neo4j version | Neo4j Graph Data Science -| `5.5` | `2.3.2 or later` +| `5.6` | `2.3.2 or later` | `5.5` | `2.3.1 or later` | `5.4` | `2.3`, `2.2.7 or later` footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] | `5.3` | `2.3`, `2.2.6 or later` footnote:eol[] From 14b76bc2348572fda84fa4374b23e22ac3597891 Mon Sep 17 00:00:00 2001 From: Brian Shi Date: Wed, 5 Apr 2023 16:15:58 +0100 Subject: [PATCH 314/400] Make splits invariant to class values --- .../classification/GlobalAccuracy.java | 3 +- .../ml/splitting/StratifiedKFoldSplitter.java | 8 +- .../gds/ml/training/CrossValidation.java | 4 +- .../gds/ml/training/TrainingStatistics.java | 5 + .../train/LinkPredictionTrain.java | 1 + .../train/NodeClassificationTrain.java | 9 +- ...ficationTrainClassValueInvarianceTest.java | 189 ++++++++++++++++++ 7 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrainClassValueInvarianceTest.java diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/metrics/classification/GlobalAccuracy.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/metrics/classification/GlobalAccuracy.java index a97c45b4950..6c1d8e5ff26 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/metrics/classification/GlobalAccuracy.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/metrics/classification/GlobalAccuracy.java @@ -24,6 +24,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Comparator; +import java.util.Objects; import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @@ -69,7 +70,7 @@ public double compute(HugeIntArray targets, HugeIntArray predictions) { @Override public int hashCode() { - return super.hashCode(); + return Objects.hash(NAME); } @Override diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/StratifiedKFoldSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/StratifiedKFoldSplitter.java index 002e0a212d0..4af699d4b62 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/StratifiedKFoldSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/StratifiedKFoldSplitter.java @@ -50,7 +50,7 @@ public class StratifiedKFoldSplitter { private final ReadOnlyHugeLongArray ids; private final LongToLongFunction targets; private final SplittableRandom random; - private final SortedSet distinctTargets; + private final SortedSet distinctInternalTargets; public static MemoryEstimation memoryEstimationForNodeSet(int k, double trainFraction) { return memoryEstimation(k, dim -> (long) (dim.nodeCount() * trainFraction)); @@ -79,12 +79,12 @@ public static MemoryEstimation memoryEstimation(int k, ToLongFunction randomSeed, SortedSet distinctTargets) { + public StratifiedKFoldSplitter(int k, ReadOnlyHugeLongArray ids, LongToLongFunction targets, Optional randomSeed, SortedSet distinctInternalTargets) { this.k = k; this.ids = ids; this.targets = targets; this.random = ShuffleUtil.createRandomDataGenerator(randomSeed); - this.distinctTargets = distinctTargets; + this.distinctInternalTargets = distinctInternalTargets; } public List splits() { @@ -97,7 +97,7 @@ public List splits() { allocateArrays(nodeCount, trainSets, testSets); var roundRobinPointer = new MutableInt(); - distinctTargets.forEach(currentClass -> { + distinctInternalTargets.forEach(currentClass -> { for (long offset = 0; offset < ids.size(); offset++) { var id = ids.get(offset); if (targets.applyAsLong(id) == currentClass) { diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/CrossValidation.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/CrossValidation.java index cb13261ea17..88b4d58e79e 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/CrossValidation.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/CrossValidation.java @@ -85,7 +85,7 @@ public CrossValidation( public void selectModel( ReadOnlyHugeLongArray outerTrainSet, LongToLongFunction targets, - SortedSet distinctTargets, + SortedSet distinctInternalTargets, TrainingStatistics trainingStatistics, Iterator modelCandidates ) { @@ -95,7 +95,7 @@ public void selectModel( outerTrainSet, targets, randomSeed, - distinctTargets + distinctInternalTargets ).splits(); progressTracker.endSubTask("Create validation folds"); diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/TrainingStatistics.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/TrainingStatistics.java index d6265ea9beb..ffbe584e17d 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/TrainingStatistics.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/training/TrainingStatistics.java @@ -60,6 +60,11 @@ public List getValidationStats(Metric metric) { return modelCandidateStats.stream().map(stats -> stats.validationStats().get(metric)).collect(Collectors.toList()); } + @TestOnly + public Double getTestScore(Metric metric) { + return testScores.get(metric); + } + /** * Turns this class into a Cypher map, to be returned in a procedure YIELD field. * This is intentionally omitting the test scores. diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrain.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrain.java index 702723025d4..6f6375ddfe0 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrain.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionTrain.java @@ -207,6 +207,7 @@ private void findBestModelCandidate( crossValidation.selectModel( trainRelationshipIds, trainData.labels()::get, + //LP always have 2 classes 0,1 the original ids happen to be the same as internal new TreeSet<>(classIdMap.originalIdsList()), trainingStatistics, modelCandidates diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrain.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrain.java index 4a3972a26a0..d3ebf2dc16d 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrain.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrain.java @@ -70,6 +70,7 @@ import java.util.TreeSet; import java.util.function.LongUnaryOperator; import java.util.stream.Collectors; +import java.util.stream.LongStream; import static org.neo4j.gds.core.utils.mem.MemoryEstimations.delegateEstimation; import static org.neo4j.gds.core.utils.mem.MemoryEstimations.maxEstimation; @@ -374,10 +375,10 @@ private void findBestModelCandidate(ReadOnlyHugeLongArray trainNodeIds, Features trainConfig.randomSeed() ); - var sortedClassIds = new TreeSet(); - for (long clazz : classCounts.keys()) { - sortedClassIds.add(clazz); - } + var sortedClassIds = LongStream + .range(0, classCounts.size()) + .boxed() + .collect(Collectors.toCollection(TreeSet::new)); crossValidation.selectModel( trainNodeIds, diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrainClassValueInvarianceTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrainClassValueInvarianceTest.java new file mode 100644 index 00000000000..3da0f72c518 --- /dev/null +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/nodePipeline/classification/train/NodeClassificationTrainClassValueInvarianceTest.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.ml.pipeline.nodePipeline.classification.train; + +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.executor.ExecutionContext; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; +import org.neo4j.gds.ml.metrics.classification.Accuracy; +import org.neo4j.gds.ml.metrics.classification.ClassificationMetricSpecification; +import org.neo4j.gds.ml.metrics.classification.GlobalAccuracy; +import org.neo4j.gds.ml.models.logisticregression.LogisticRegressionTrainConfigImpl; +import org.neo4j.gds.ml.pipeline.nodePipeline.NodeFeatureProducer; +import org.neo4j.gds.ml.pipeline.nodePipeline.NodeFeatureStep; +import org.neo4j.gds.ml.pipeline.nodePipeline.classification.NodeClassificationTrainingPipeline; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@GdlExtension +public class NodeClassificationTrainClassValueInvarianceTest { + + private static final String GRAPH_NAME_1 = "G11"; + + @GdlGraph(graphNamePrefix = "nodes1") + private static final String DB_QUERY1 = + "CREATE " + + " (a1:N {bananas: 100.0, arrayProperty: [1.2, 1.2], a: 1.2, b: 1.2, t: 0})" + + ", (a2:N {bananas: 100.0, arrayProperty: [2.8, 2.5], a: 2.8, b: 2.5, t: 0})" + + ", (a3:N {bananas: 100.0, arrayProperty: [3.3, 0.5], a: 3.3, b: 0.5, t: 0})" + + ", (a4:N {bananas: 100.0, arrayProperty: [1.0, 0.5], a: 1.0, b: 0.5, t: 0})" + + ", (a5:N {bananas: 100.0, arrayProperty: [1.32, 0.5], a: 1.32, b: 0.5, t: 0})" + + ", (a6:N {bananas: 100.0, arrayProperty: [1.3, 1.5], a: 1.3, b: 1.5, t: 1})" + + ", (a7:N {bananas: 100.0, arrayProperty: [5.3, 10.5], a: 5.3, b: 10.5, t: 1})" + + ", (a8:N {bananas: 100.0, arrayProperty: [1.3, 2.5], a: 1.3, b: 2.5, t: 1})" + + ", (a9:N {bananas: 100.0, arrayProperty: [0.0, 66.8], a: 0.0, b: 66.8, t: 1})" + + ", (a10:N {bananas: 100.0, arrayProperty: [0.1, 2.8], a: 0.1, b: 2.8, t: 1})" + + ", (a11:N {bananas: 100.0, arrayProperty: [0.66, 2.8], a: 0.66, b: 2.8, t: 1})" + + ", (a12:N {bananas: 100.0, arrayProperty: [2.0, 10.8], a: 2.0, b: 10.8, t: 1})" + + ", (a13:N {bananas: 100.0, arrayProperty: [5.0, 7.8], a: 5.0, b: 7.8, t: 2})" + + ", (a14:N {bananas: 100.0, arrayProperty: [4.0, 5.8], a: 4.0, b: 5.8, t: 2})" + + ", (a15:N {bananas: 100.0, arrayProperty: [1.0, 0.9], a: 1.0, b: 0.9, t: 2})"; + + @Inject + private GraphStore nodes1GraphStore; + + private static final String GRAPH_NAME_2 = "G2"; + + @GdlGraph(graphNamePrefix = "nodes2") + private static final String DB_QUERY2 = + "CREATE " + + " (a1:N {bananas: 100.0, arrayProperty: [1.2, 1.2], a: 1.2, b: 1.2, t: 0})" + + ", (a2:N {bananas: 100.0, arrayProperty: [2.8, 2.5], a: 2.8, b: 2.5, t: 0})" + + ", (a3:N {bananas: 100.0, arrayProperty: [3.3, 0.5], a: 3.3, b: 0.5, t: 0})" + + ", (a4:N {bananas: 100.0, arrayProperty: [1.0, 0.5], a: 1.0, b: 0.5, t: 0})" + + ", (a5:N {bananas: 100.0, arrayProperty: [1.32, 0.5], a: 1.32, b: 0.5, t: 0})" + + ", (a6:N {bananas: 100.0, arrayProperty: [1.3, 1.5], a: 1.3, b: 1.5, t: 222})" + + ", (a7:N {bananas: 100.0, arrayProperty: [5.3, 10.5], a: 5.3, b: 10.5, t: 222})" + + ", (a8:N {bananas: 100.0, arrayProperty: [1.3, 2.5], a: 1.3, b: 2.5, t: 222})" + + ", (a9:N {bananas: 100.0, arrayProperty: [0.0, 66.8], a: 0.0, b: 66.8, t: 222})" + + ", (a10:N {bananas: 100.0, arrayProperty: [0.1, 2.8], a: 0.1, b: 2.8, t: 222})" + + ", (a11:N {bananas: 100.0, arrayProperty: [0.66, 2.8], a: 0.66, b: 2.8, t: 222})" + + ", (a12:N {bananas: 100.0, arrayProperty: [2.0, 10.8], a: 2.0, b: 10.8, t: 222})" + + ", (a13:N {bananas: 100.0, arrayProperty: [5.0, 7.8], a: 5.0, b: 7.8, t: 333})" + + ", (a14:N {bananas: 100.0, arrayProperty: [4.0, 5.8], a: 4.0, b: 5.8, t: 333})" + + ", (a15:N {bananas: 100.0, arrayProperty: [1.0, 0.9], a: 1.0, b: 0.9, t: 333})"; + + @Inject + private GraphStore nodes2GraphStore; + + /** + * This tests that the specific class values do not matter, as long as the ordering is the same. + * However, if the *ordering* of the class values differ, the splits in cross-validation could differ, resulting in different accuracies. + */ + @Test + void trainWithDifferentClassValues() { + var pipeline = new NodeClassificationTrainingPipeline(); + pipeline.addFeatureStep(NodeFeatureStep.of("a")); + pipeline.addFeatureStep(NodeFeatureStep.of("b")); + + var lrTrainerConfig = LogisticRegressionTrainConfigImpl.builder().build(); + pipeline.addTrainerConfig(lrTrainerConfig); + + var accuracyMetricSpec = ClassificationMetricSpecification.Parser.parse("accuracy"); + var accuracyPerClassMetricSpec = ClassificationMetricSpecification.Parser.parse("accuracy(class=*)"); + + var config01 = createConfig("model1", GRAPH_NAME_1, List.of(accuracyMetricSpec, accuracyPerClassMetricSpec), 1L); + var ncTrain01 = createWithExecutionContext( + nodes1GraphStore, + pipeline, + config01, + ProgressTracker.NULL_TRACKER + ); + var result01 = ncTrain01.run(); + assertThat(result01.classifier().data().featureDimension()).isEqualTo(2); + + //Run with graph that have class values 0 and 2 + var config02 = createConfig("model2", GRAPH_NAME_2, List.of(accuracyMetricSpec, accuracyPerClassMetricSpec), 1L); + var ncTrain02 = createWithExecutionContext( + nodes2GraphStore, + pipeline, + config02, + ProgressTracker.NULL_TRACKER + ); + var result02 = ncTrain02.run(); + assertThat(result01.classifier().data().featureDimension()).isEqualTo(2); + + var globalAccuracy = new GlobalAccuracy(); + var accuracyForClass1 = new Accuracy(0, 0); + var accuracyForClass2 = new Accuracy(1, 1); + var accuracyForClass3 = new Accuracy(2, 2); + + var accuracyForClass0 = new Accuracy(0, 0); + var accuracyForClass222 = new Accuracy(222, 1); + var accuracyForClass333 = new Accuracy(333, 2); + + assertThat(result01.trainingStatistics().getTrainStats(globalAccuracy)).isEqualTo(result02.trainingStatistics().getTrainStats(globalAccuracy)); + assertThat(result01.trainingStatistics().getTrainStats(accuracyForClass1)).isEqualTo(result02.trainingStatistics().getTrainStats(accuracyForClass0)); + assertThat(result01.trainingStatistics().getTrainStats(accuracyForClass2)).isEqualTo(result02.trainingStatistics().getTrainStats(accuracyForClass222)); + assertThat(result01.trainingStatistics().getTrainStats(accuracyForClass3)).isEqualTo(result02.trainingStatistics().getTrainStats(accuracyForClass333)); + + assertThat(result01.trainingStatistics().getValidationStats(globalAccuracy)).isEqualTo(result02.trainingStatistics().getValidationStats(globalAccuracy)); + assertThat(result01.trainingStatistics().getValidationStats(accuracyForClass1)).isEqualTo(result02.trainingStatistics().getValidationStats(accuracyForClass0)); + assertThat(result01.trainingStatistics().getValidationStats(accuracyForClass2)).isEqualTo(result02.trainingStatistics().getValidationStats(accuracyForClass222)); + assertThat(result01.trainingStatistics().getValidationStats(accuracyForClass3)).isEqualTo(result02.trainingStatistics().getValidationStats(accuracyForClass333)); + + assertThat(result01.trainingStatistics().getTestScore(globalAccuracy)).isEqualTo(result02.trainingStatistics().getTestScore(globalAccuracy)); + assertThat(result01.trainingStatistics().getTestScore(accuracyForClass1)).isEqualTo(result02.trainingStatistics().getTestScore(accuracyForClass0)); + assertThat(result01.trainingStatistics().getTestScore(accuracyForClass2)).isEqualTo(result02.trainingStatistics().getTestScore(accuracyForClass222)); + assertThat(result01.trainingStatistics().getTestScore(accuracyForClass3)).isEqualTo(result02.trainingStatistics().getTestScore(accuracyForClass333)); + + } + + private NodeClassificationPipelineTrainConfig createConfig( + String modelName, + String graphName, + List metricSpecification, + long randomSeed + ) { + return NodeClassificationPipelineTrainConfigImpl.builder() + .pipeline("") + .graphName(graphName) + .modelUser("DUMMY") + .modelName(modelName) + .concurrency(1) + .randomSeed(randomSeed) + .targetProperty("t") + .metrics(metricSpecification) + .build(); + } + + static NodeClassificationTrain createWithExecutionContext( + GraphStore graphStore, + NodeClassificationTrainingPipeline pipeline, + NodeClassificationPipelineTrainConfig config, + ProgressTracker progressTracker + ) { + var nodeFeatureProducer = NodeFeatureProducer.create(graphStore, config, ExecutionContext.EMPTY, progressTracker); + return NodeClassificationTrain.create( + graphStore, + pipeline, + config, + nodeFeatureProducer, + progressTracker + ); + } + +} From cc8d92dbc8440a15b3c00fde689b28d61b615ed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 11 Apr 2023 13:39:59 +0200 Subject: [PATCH 315/400] Publish 5.7 compat --- .../_56/InMemoryStorageEngineFactory.java | 2 +- .../5.7/neo4j-kernel-adapter/build.gradle | 63 ++ .../gds/compat/_57/Neo4jProxyFactoryImpl.java | 44 + .../compat/_57/SettingProxyFactoryImpl.java | 44 + .../compat/_57/BoltTransactionRunnerImpl.java | 98 ++ .../gds/compat/_57/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_57/CompatAccessModeImpl.java | 44 + .../_57/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_57/CompatIndexQueryImpl.java | 31 + .../_57/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_57/CompositeNodeCursorImpl.java | 32 + .../gds/compat/_57/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../gds/compat/_57/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_57/Neo4jProxyImpl.java | 923 ++++++++++++++++++ .../compat/_57/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_57/PartitionedStoreScan.java | 58 ++ .../_57/ReferencePropertyReference.java | 49 + .../compat/_57/ScanBasedStoreScanImpl.java | 40 + .../compat/_57/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_57/SettingProxyImpl.java | 87 ++ .../org/neo4j/gds/compat/_57/TestLogImpl.java | 146 +++ .../compat/_57/VirtualRelationshipImpl.java | 41 + .../5.7/storage-engine-adapter/build.gradle | 66 ++ .../_57/InMemoryStorageEngineFactory.java | 268 +++++ .../_57/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_57/InMemoryCountsStoreImpl.java | 112 +++ .../_57/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_57/InMemoryNodeCursor.java | 82 ++ .../_57/InMemoryNodePropertyCursor.java | 45 + .../compat/_57/InMemoryPropertyCursor.java | 71 ++ .../_57/InMemoryPropertySelectionImpl.java | 55 ++ .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_57/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_57/InMemoryStorageEngineFactory.java | 558 +++++++++++ .../compat/_57/InMemoryStorageEngineImpl.java | 294 ++++++ .../compat/_57/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_57/InMemoryStoreVersion.java | 68 ++ .../_57/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_57/InMemoryVersionCheck.java | 61 ++ .../_57/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_57/StorageEngineProxyImpl.java | 156 +++ .../InMemoryLogVersionRepository57.java | 71 ++ ...InMemoryStorageCommandReaderFactory57.java | 43 + .../InMemoryStorageReader57.java | 332 +++++++ gradle/dependencies.gradle | 1 + .../org/neo4j/gds/compat/Neo4jVersion.java | 7 +- .../neo4j/gds/compat/Neo4jVersionTest.java | 4 +- .../java/org/neo4j/gds/SysInfoProcTest.java | 13 + 52 files changed, 5272 insertions(+), 3 deletions(-) create mode 100644 compatibility/5.7/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableProcedureImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatAccessModeImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatIndexQueryImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/PartitionedStoreScan.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ReferencePropertyReference.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/TestLogImpl.java create mode 100644 compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/VirtualRelationshipImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodeCursor.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertyCursor.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStoreVersion.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryVersionCheck.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository57.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory57.java create mode 100644 compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader57.java diff --git a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java index c95775df36d..e489afc7173 100644 --- a/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java +++ b/compatibility/5.6/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_56/InMemoryStorageEngineFactory.java @@ -114,7 +114,7 @@ @ServiceProvider public class InMemoryStorageEngineFactory implements StorageEngineFactory { - static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-rc"; + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-56"; public InMemoryStorageEngineFactory() { StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_6, StorageEngineFactory.class); diff --git a/compatibility/5.7/neo4j-kernel-adapter/build.gradle b/compatibility/5.7/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..c80519fa5ac --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.7' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.7' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.7' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.7' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.7' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.7' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.7' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..48377e059a3 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.7 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.7 (placeholder)"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..6062d17edd2 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.7 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.7 (placeholder)"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/BoltTransactionRunnerImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..6a4b13ca2c1 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/BoltTransactionRunnerImpl.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.bookmark.Bookmark; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableProcedureImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableProcedureImpl.java new file mode 100644 index 00000000000..9729407a4bd --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableUserAggregationFunctionImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..52a05063eb1 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatAccessModeImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatAccessModeImpl.java new file mode 100644 index 00000000000..7f009169ce0 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatGraphDatabaseAPIImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..6c9b26e3f2a --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatIndexQueryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..423c2908fd8 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatUsernameAuthSubjectImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..bf183d390c0 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompositeNodeCursorImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..968b165ee3b --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseLayoutImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..619c6359717 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..25ebefcf2b7 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..4b88ffcb482 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_7; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.7"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java new file mode 100644 index 00000000000..dbc5e0e2c59 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java @@ -0,0 +1,923 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/NodeLabelIndexLookupImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..800f5875918 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/PartitionedStoreScan.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/PartitionedStoreScan.java new file mode 100644 index 00000000000..52263aec779 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ReferencePropertyReference.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ReferencePropertyReference.java new file mode 100644 index 00000000000..d7d4eb51c01 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ScanBasedStoreScanImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..13b6951eebd --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..a24ff5dc141 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_7; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.7"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyImpl.java new file mode 100644 index 00000000000..59d32e65627 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/TestLogImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/TestLogImpl.java new file mode 100644 index 00000000000..d6bf40e9819 --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/VirtualRelationshipImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..96e1eae203a --- /dev/null +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/build.gradle b/compatibility/5.7/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..d12c369acf3 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.7' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.7' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.7' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.7' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.7' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.7' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java b/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..ad053e0baee --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported57"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..76b6f44b528 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.7 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.7"; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCommandCreationContextImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..11f042220e0 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCountsStoreImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..3f6f64ecaa5 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryMetaDataProviderImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..42a8f5a3576 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository57; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository57 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository57(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodeCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodeCursor.java new file mode 100644 index 00000000000..c41c50fa13e --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodePropertyCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..1a18e5ff264 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertyCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..693f57da32b --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertySelectionImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..3ab5d56cc97 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipPropertyCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..34befce80f6 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipScanCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..332dd46668d --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipTraversalCursor.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..7ac42153d33 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..cbace6c2601 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineFactory.java @@ -0,0 +1,558 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory57; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-57"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_7, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory57.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..96b8f6a8ce9 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageEngineImpl.java @@ -0,0 +1,294 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader57; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private final StorageEngineIndexingBehaviour indexingBehaviour = () -> false; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return indexingBehaviour; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader57(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageLocksImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..aa89bea2c72 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStoreVersion.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStoreVersion.java new file mode 100644 index 00000000000..3ab0f1cf00e --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryTransactionIdStoreImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..976995f6ad2 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryVersionCheck.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryVersionCheck.java new file mode 100644 index 00000000000..18ea10e4da9 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._57.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..67197cacc4e --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_7; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.7"; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..8a0eadab4b5 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_57/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._57; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader57; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader57(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository57.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository57.java new file mode 100644 index 00000000000..7fb6d12a43a --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository57.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository57 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository57() { + this(0, 0); + } + + private InMemoryLogVersionRepository57(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory57.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory57.java new file mode 100644 index 00000000000..02fdf497d40 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory57.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory57 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory57(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader57.java b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader57.java new file mode 100644 index 00000000000..1d64a8e97c1 --- /dev/null +++ b/compatibility/5.7/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader57.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._57.InMemoryNodeCursor; +import org.neo4j.gds.compat._57.InMemoryPropertyCursor; +import org.neo4j.gds.compat._57.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._57.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader57 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader57( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index b693a0d6bd1..6bdc226e98c 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -7,6 +7,7 @@ ext { '5.4': properties.getOrDefault('neo4jVersion54', '5.4.0'), '5.5': properties.getOrDefault('neo4jVersion55', '5.5.0'), '5.6': properties.getOrDefault('neo4jVersion56', '5.6.0'), + '5.7': properties.getOrDefault('neo4jVersion57', '5.7.0'), ] neo4jDefault = neos.'4.4' diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 473aa513dd3..2593a05bd1d 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -34,6 +34,7 @@ public enum Neo4jVersion { V_5_4, V_5_5, V_5_6, + V_5_7, V_RC; @Override @@ -53,6 +54,8 @@ public String toString() { return "5.5"; case V_5_6: return "5.6"; + case V_5_7: + return "5.7"; case V_RC: return "rc"; default: @@ -62,7 +65,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 7); + return ImmutableMajorMinorVersion.of(5, 8); } String version = toString(); var subVersions = version.split("\\."); @@ -137,6 +140,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 6) { return Neo4jVersion.V_5_6; } else if (minorVersion == 7) { + return Neo4jVersion.V_5_7; + } else if (minorVersion == 8) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 0989f6bad2b..3b940016f6a 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -47,7 +47,8 @@ class Neo4jVersionTest { "5.4.0, V_5_4", "5.5.0, V_5_5", "5.6.0, V_5_6", - "5.7.0, V_RC", + "5.7.0, V_5_7", + "5.8.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); @@ -86,6 +87,7 @@ void shouldNotRespectVersionOverride() { "5.4.0, 5, 4", "5.5.0, 5, 5", "5.6.0, 5, 6", + "5.7.0, 5, 7", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index d2c3b78231e..69895a17779 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -81,6 +81,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.6", "Neo4j Settings 5.6 (placeholder)", + "Neo4j 5.7", + "Neo4j 5.7 (placeholder)", + "Neo4j Settings 5.7", + "Neo4j Settings 5.7 (placeholder)", + "Neo4j RC", "Neo4j RC (placeholder)", "Neo4j Settings RC", @@ -184,6 +189,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.6" ); break; + case V_5_7: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.7 (placeholder)", + "Neo4j Settings 5.7", + "Neo4j 5.7 (placeholder)", + "Neo4j 5.7" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC (placeholder)", From 300219743d7d4b16c9691e1903b244cc82935340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 12 Apr 2023 14:47:55 +0200 Subject: [PATCH 316/400] Bump AuraDS build number --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 7aa39f5d851..22b877c2e52 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.3' - gdsAuraVersion = '16' + gdsAuraVersion = '17' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 0cb331b60893daa661e891eff4d762cec2a883ed Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Wed, 12 Apr 2023 14:50:59 +0100 Subject: [PATCH 317/400] Clarify installation instructions --- doc/modules/ROOT/pages/installation/index.adoc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/index.adoc b/doc/modules/ROOT/pages/installation/index.adoc index e3cbbb664cd..b0d609db913 100644 --- a/doc/modules/ROOT/pages/installation/index.adoc +++ b/doc/modules/ROOT/pages/installation/index.adoc @@ -5,10 +5,7 @@ The Neo4j Graph Data Science (GDS) library is delivered as a plugin to the Neo4j Graph Database. The plugin needs to be installed into the database and added to the allowlist in the Neo4j configuration. -There are two main ways of achieving this, which we will detail in this chapter. - - -This chapter is divided into the following sections: +The following sections cover the different installation methods. . xref:installation/supported-neo4j-versions.adoc[Supported Neo4j versions] . xref:installation/neo4j-desktop.adoc[Neo4j Desktop] From 1fa79d3a6faf5ffc94bbe91abc1589139df998d4 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Wed, 12 Apr 2023 15:00:41 +0100 Subject: [PATCH 318/400] Apply suggestions from code review Co-authored-by: Lidia Zuin <102308961+lidiazuin@users.noreply.github.com> --- doc/modules/ROOT/pages/installation/index.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/installation/index.adoc b/doc/modules/ROOT/pages/installation/index.adoc index b0d609db913..84e9a30df48 100644 --- a/doc/modules/ROOT/pages/installation/index.adoc +++ b/doc/modules/ROOT/pages/installation/index.adoc @@ -5,7 +5,7 @@ The Neo4j Graph Data Science (GDS) library is delivered as a plugin to the Neo4j Graph Database. The plugin needs to be installed into the database and added to the allowlist in the Neo4j configuration. -The following sections cover the different installation methods. +The following sections cover the different installation methods: . xref:installation/supported-neo4j-versions.adoc[Supported Neo4j versions] . xref:installation/neo4j-desktop.adoc[Neo4j Desktop] From ae460ff46fd928bbfdb092f006c04a616027c735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 5 Apr 2023 13:15:03 +0200 Subject: [PATCH 319/400] Improve cypher aggregation docs * how labels and properties are picked * parameters inside the node and relationshipConfig * fix the optional parameters intro --- .../graph-project-cypher-aggregation.adoc | 58 ++++++++++++++----- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc index 254c3f1ab16..5cf8066f440 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc @@ -3,22 +3,17 @@ :description: This section details projecting GDS graphs using `Cypher` aggregations. +Using Cypher aggregations is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. +Cypher aggregations are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). -A projected graph can be stored in the catalog under a user-defined name. -Using that name, the graph can be referred to by any algorithm in the library. -This allows multiple algorithms to use the same graph without having to project it on each algorithm run. -Using Cypher aggregations is a more flexible and expressive approach with diminished focus on performance compared to the xref:management-ops/projections/graph-project.adoc[native projections]. -Cypher projections are primarily recommended for the development phase (see xref:common-usage/index.adoc[Common usage]). +== Considerations -[NOTE] --- -There is also a way to generate a random graph, see xref:management-ops/projections/graph-generation.adoc[Graph Generation] documentation for more details. --- +=== Lifecycle [NOTE] -- -The projected graph will reside in the catalog until: +The projected graphs will reside in the catalog until either: - the graph is dropped using xref:graph-drop.adoc[gds.graph.drop] - the Neo4j database from which the graph was projected is stopped or dropped @@ -26,12 +21,25 @@ The projected graph will reside in the catalog until: -- +=== Node property support + +Cypher aggregations can only project a limited set of node property types from a Cypher query. +The xref:management-ops/node-properties.adoc#node-properties-supported[Node Properties page] details which node property types are supported. +Other types of node properties have to be transformed or encoded into one of the supported types in order to be projected using a Cypher aggregation. + +=== Selection of node properties and labels + +If a node occurs multiple times, the node properties and labels of the first occurrence will be used for the projection. +This is important when a node can be a source node as well as a target node. + + [[graph-project-cypher-aggregation-syntax]] == Syntax A Cypher aggregation is used in a query as an aggregation over the relationships that are being projected. It takes three mandatory arguments: `graphName`, `sourceNode` and `targetNode`. -In addition, the optional `sourceNodeProperties`, `targetNodeProperties`, and `relationshipProperties` parameters allows us to project properties. +In addition, the optional `nodesConfig`, `relationshipConfig` parameters can be used to project properties and label or types. +The optional `configuration` parameter can be used for general configuration of the projection such as `readConcurrency`. [.graph-project-cypher-aggregation-syntax] -- @@ -53,15 +61,35 @@ RETURN gds.alpha.graph.project( ---- .Parameters -[opts="header",cols="1,1,8"] +[opts="header",cols="2,1,7"] |=== | Name | Optional | Description | graphName | no | The name under which the graph is stored in the catalog. | sourceNode | no | The source node of the relationship. Must not be null. | targetNode | yes | The target node of the relationship. The targetNode can be null (for example due to an `OPTIONAL MATCH`), in which case the source node is projected as an unconnected node. -| nodesConfig | yes | Properties and Labels configuration for the source and target nodes. -| relationshipConfig | yes | Properties and Type configuration for the relationship. -| configuration | yes | Additional parameters to configure the cypher aggregation projection. +| <> | yes | Properties and Labels configuration for the source and target nodes. +| <> | yes | Properties and Type configuration for the relationship. +| <> | yes | Additional parameters to configure the cypher aggregation projection. +|=== + +[[graph-project-cypher-aggregation-syntax-nodesConfig]] +.Nodes configuration +[opts="header",cols="1,1,1,4"] +|=== +| Name | Type | Default | Description +| sourceNodeProperties | Map | {} | The properties of the source node. +| targetNodeProperties | Map | {} | The properties of the target node. +| sourceNodeLabels | List of String or String | [] | The label(s) of the source node. +| targetNodeLabels | List of String or String | [] | The label(s) of the source node. +|=== + +[[graph-project-cypher-aggregation-syntax-relationshipConfig]] +.Relationship configuration +[opts="header",cols="1,1,1,4"] +|=== +| Name | Type | Default | Description +| properties | Map | {} | The properties of the source node. +| relationshipType | String | '*' | The type of the relationship. |=== [[graph-project-cypher-aggregation-syntax-configuration]] From c464497c1c7d492a5104681a8243ea6383b256e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 13 Apr 2023 12:49:27 +0200 Subject: [PATCH 320/400] Handle links inside proc parameters in syntax doc test --- .../syntax/ProcedureSyntaxAutoChecker.java | 4 +- .../ProcedureSyntaxAutoCheckerTest.java | 21 ++++++---- ...clude-with-syntax-parameter-with-link.adoc | 42 +++++++++++++++++++ 3 files changed, 59 insertions(+), 8 deletions(-) create mode 100644 doc-test-tools/src/test/resources/include-with-syntax-parameter-with-link.adoc diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoChecker.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoChecker.java index baab4ddbd7c..50218ee7fc0 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoChecker.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoChecker.java @@ -166,6 +166,8 @@ private Consumer assertTableValues( .getCells() .get(0)) // Get the first column in the row --> corresponds to the return column names .map(Cell::getText) + // remove any potential links in the names + .map(name -> name.replaceAll("|<\\/a>", "")) // as java identifier cannot contain white spaces, remove anything after the first space such as footnote: .map(name -> name.split("\\s+")[0]) .collect(Collectors.toList()); @@ -176,7 +178,7 @@ private Consumer assertTableValues( }; } - private Iterable extractDocResultFields(String syntaxCode) { + private static Iterable extractDocResultFields(String syntaxCode) { var yield = syntaxCode.substring(syntaxCode.indexOf(YIELD_KEYWORD) + YIELD_KEYWORD.length()).trim(); return Arrays.stream(yield.split(YIELD_FIELD_SEPARATOR)) .map(yieldField -> yieldField.split(YIELD_NAME_DATA_TYPE_SEPARATOR)[0].trim()) diff --git a/doc-test-tools/src/test/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoCheckerTest.java b/doc-test-tools/src/test/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoCheckerTest.java index 9e439cce525..5d9838e0e65 100644 --- a/doc-test-tools/src/test/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoCheckerTest.java +++ b/doc-test-tools/src/test/java/org/neo4j/gds/doc/syntax/ProcedureSyntaxAutoCheckerTest.java @@ -23,11 +23,14 @@ import org.asciidoctor.OptionsBuilder; import org.asciidoctor.SafeMode; import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.File; import java.net.URISyntaxException; @@ -48,6 +51,9 @@ class ProcedureSyntaxAutoCheckerTest { private static final String newLine = System.lineSeparator(); + @InjectSoftAssertions + private SoftAssertions softAssertions; + @BeforeEach void setUp() { // By default we are forced to use relative path which we don't want. @@ -57,10 +63,11 @@ void setUp() { } - @Test - void correctSyntaxSectionTest(SoftAssertions softAssertions) throws URISyntaxException { + @ParameterizedTest(name = "{0}") + @ValueSource(strings = {"include-with-syntax.adoc", "include-with-syntax-parameter-with-link.adoc"}) + void correctSyntaxSectionTest(String positiveResource) throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { - var file = Paths.get(getClass().getClassLoader().getResource("include-with-syntax.adoc").toURI()).toFile(); + var file = Paths.get(getClass().getClassLoader().getResource(positiveResource).toURI()).toFile(); assertTrue(file.exists() && file.canRead()); asciidoctor.convertFile(file, options); @@ -68,7 +75,7 @@ void correctSyntaxSectionTest(SoftAssertions softAssertions) throws URISyntaxExc } @Test - void shouldFailOnMissingResultsTable(SoftAssertions softAssertions) throws URISyntaxException { + void shouldFailOnMissingResultsTable() throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { var file = Paths .get(getClass() @@ -85,7 +92,7 @@ void shouldFailOnMissingResultsTable(SoftAssertions softAssertions) throws URISy } @Test - void shouldFailOnMoreThanOneResultsTable(SoftAssertions softAssertions) throws URISyntaxException { + void shouldFailOnMoreThanOneResultsTable() throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { var file = Paths .get(getClass() @@ -102,7 +109,7 @@ void shouldFailOnMoreThanOneResultsTable(SoftAssertions softAssertions) throws U } @Test - void shouldFailOnMissingCodeBlock(SoftAssertions softAssertions) throws URISyntaxException { + void shouldFailOnMissingCodeBlock() throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { var file = Paths .get(getClass().getClassLoader().getResource("invalid-include-with-syntax-no-code-block.adoc").toURI()) @@ -116,7 +123,7 @@ void shouldFailOnMissingCodeBlock(SoftAssertions softAssertions) throws URISynta } @Test - void shouldFailOnMoreThanOneCodeBlock(SoftAssertions softAssertions) throws URISyntaxException { + void shouldFailOnMoreThanOneCodeBlock() throws URISyntaxException { try (var asciidoctor = createAsciidoctor(softAssertions)) { var file = Paths .get(getClass() diff --git a/doc-test-tools/src/test/resources/include-with-syntax-parameter-with-link.adoc b/doc-test-tools/src/test/resources/include-with-syntax-parameter-with-link.adoc new file mode 100644 index 00000000000..4d1f95c7d18 --- /dev/null +++ b/doc-test-tools/src/test/resources/include-with-syntax-parameter-with-link.adoc @@ -0,0 +1,42 @@ +[.include-with-stream] +====== +.Run Louvain in stream mode on a named graph. +[source, cypher, role=noplay] +---- +CALL gds.louvain.stream( + graphName: String, + configuration: Map +) +YIELD + nodeId: Integer, + communityId: Integer, + intermediateCommunityIds: List of Integer +---- + +.Parameters +[opts="header"] +|=== +| Name +| graphName footnote:vital[choose the name wisely] +| <> +|=== + +// This table is only here to make sure we will really pick the `.Results` one +[[configuration-table]] +.Algorithm specific configuration +[opts="header",cols="1,1,1m,1,4"] +|=== +| Name | Type | Default | Optional | Description +| relationshipWeightProperty | String | null | yes | Relationship Weight. +| seedProperty | String | n/a | yes | Seed Property. +|=== + +.Results +[opts="header",cols="1,1,6"] +|=== +| Name | Type | Description +| nodeId | Integer | Node ID. +| communityId | Integer | The community ID of the final level. +| intermediateCommunityIds | List of Integer | Community IDs for each level. `Null` if `includeIntermediateCommunities` is set to false. +|=== +====== From 861ac3f2e2541ff1ee51fc8cc865bc6489edfb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 13 Apr 2023 15:34:50 +0200 Subject: [PATCH 321/400] Clarify how the properties and labels can differ And some smaller rewording Co-authored-by: Nicola Vitucci --- .../projections/graph-project-cypher-aggregation.adoc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc index 5cf8066f440..a4c6412c1d9 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-project-cypher-aggregation.adoc @@ -30,7 +30,8 @@ Other types of node properties have to be transformed or encoded into one of the === Selection of node properties and labels If a node occurs multiple times, the node properties and labels of the first occurrence will be used for the projection. -This is important when a node can be a source node as well as a target node. +This is important when a node can be a source node as well as a target node and their configuration differs. +Relevant configuration options are `sourceNodeProperties`, `targetNodeProperties`, `sourceNodeLabels` and `targetNodeLabels`. [[graph-project-cypher-aggregation-syntax]] @@ -38,7 +39,7 @@ This is important when a node can be a source node as well as a target node. A Cypher aggregation is used in a query as an aggregation over the relationships that are being projected. It takes three mandatory arguments: `graphName`, `sourceNode` and `targetNode`. -In addition, the optional `nodesConfig`, `relationshipConfig` parameters can be used to project properties and label or types. +In addition, two optional parameters can be used to project node properties and labels (`nodesConfig`) or relationship properties and type (`relationshipConfig`). The optional `configuration` parameter can be used for general configuration of the projection such as `readConcurrency`. [.graph-project-cypher-aggregation-syntax] @@ -67,9 +68,9 @@ RETURN gds.alpha.graph.project( | graphName | no | The name under which the graph is stored in the catalog. | sourceNode | no | The source node of the relationship. Must not be null. | targetNode | yes | The target node of the relationship. The targetNode can be null (for example due to an `OPTIONAL MATCH`), in which case the source node is projected as an unconnected node. -| <> | yes | Properties and Labels configuration for the source and target nodes. -| <> | yes | Properties and Type configuration for the relationship. -| <> | yes | Additional parameters to configure the cypher aggregation projection. +| <> | yes | Properties and labels configuration for the source and target nodes. +| <> | yes | Properties and type configuration for the relationship. +| <> | yes | Additional parameters to configure the projection. |=== [[graph-project-cypher-aggregation-syntax-nodesConfig]] From 624bd684e7f5583fd2ecaba6a36de0f109956b34 Mon Sep 17 00:00:00 2001 From: Aaron Hiniker Date: Fri, 24 Mar 2023 14:35:29 -0700 Subject: [PATCH 322/400] RandomWalk: ensure we read updated termination flag in algorithm --- .../org/neo4j/gds/traversal/RandomWalk.java | 10 +++--- .../neo4j/gds/traversal/RandomWalkTest.java | 33 +++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java index 3f07c4b0993..c8e24d8deb6 100644 --- a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java +++ b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java @@ -96,7 +96,7 @@ public Stream compute() { ? new NextNodeSupplier.GraphNodeSupplier(graph.nodeCount()) : NextNodeSupplier.ListNodeSupplier.of(config, graph); - var terminationFlag = new ExternalTerminationFlag(this.terminationFlag); + var terminationFlag = new ExternalTerminationFlag(this); BlockingQueue walks = new ArrayBlockingQueue<>(config.walkBufferSize()); long[] TOMB = new long[0]; @@ -198,15 +198,15 @@ private Stream walksQueueConsumer( private static final class ExternalTerminationFlag implements TerminationFlag { private volatile boolean running = true; - private final TerminationFlag inner; + private final Algorithm algo; - ExternalTerminationFlag(TerminationFlag inner) { - this.inner = inner; + ExternalTerminationFlag(Algorithm algo) { + this.algo = algo; } @Override public boolean running() { - return this.running && this.inner.running(); + return this.running && this.algo.getTerminationFlag().running(); } void stop() { diff --git a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java index 47ad8d3fda9..ddc8b7d5fbf 100644 --- a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java +++ b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java @@ -36,6 +36,7 @@ import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.utils.TerminationFlag; import org.neo4j.gds.core.utils.progress.GlobalTaskStore; import org.neo4j.gds.core.utils.progress.TaskRegistryFactory; import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; @@ -411,6 +412,38 @@ void testWithConfiguredOffsetStartNodes() { .anyMatch(walk -> walk[0] == bInternalId); } + /** + * Ensure that when termination flag is set externally, we terminate the walk + * @throws InterruptedException + */ + @Test + void testPartialReadMultipleRuns() { + for (int i = 0; i < 3; i++) { + Node2VecStreamConfig config = ImmutableNode2VecStreamConfig.builder() + .walkBufferSize(1) + .build(); + + var randomWalk = RandomWalk.create( + graph, + config, + ProgressTracker.NULL_TRACKER, + Pools.DEFAULT + ); + + var stream = randomWalk.compute(); + long count = stream.limit(10).count(); + + randomWalk.setTerminationFlag(new TerminationFlag() { + @Override + public boolean running() { + return false; + } + }); + + assertEquals(10, count); + } + } + @Nested class ProgressTracking { From af4c7e9060f088e2e350a3cca4fe4b3bbdffd226 Mon Sep 17 00:00:00 2001 From: Aaron Hiniker Date: Fri, 24 Mar 2023 14:36:55 -0700 Subject: [PATCH 323/400] Update test --- algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java index ddc8b7d5fbf..8bf408cc79c 100644 --- a/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java +++ b/algo/src/test/java/org/neo4j/gds/traversal/RandomWalkTest.java @@ -414,10 +414,9 @@ void testWithConfiguredOffsetStartNodes() { /** * Ensure that when termination flag is set externally, we terminate the walk - * @throws InterruptedException */ @Test - void testPartialReadMultipleRuns() { + void testSetTerminationFlagAndMultipleRuns() { for (int i = 0; i < 3; i++) { Node2VecStreamConfig config = ImmutableNode2VecStreamConfig.builder() .walkBufferSize(1) From 8e73d8418f08c727f2e24e1a1a41e1a81a0d7cbd Mon Sep 17 00:00:00 2001 From: Aaron Hiniker Date: Fri, 24 Mar 2023 15:04:06 -0700 Subject: [PATCH 324/400] Allow termination in RandomWalk while filling buffer --- .../java/org/neo4j/gds/traversal/RandomWalk.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java index c8e24d8deb6..179210d398a 100644 --- a/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java +++ b/algo/src/main/java/org/neo4j/gds/traversal/RandomWalk.java @@ -38,6 +38,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -178,7 +179,10 @@ private void tasksRunner( progressTracker.endSubTask("create walks"); try { - walks.put(tombstone); + boolean finished = false; + while (!finished && terminationFlag.running()) { + finished = walks.offer(tombstone, 100, TimeUnit.MILLISECONDS); + } } catch (InterruptedException exception) { Thread.currentThread().interrupt(); } @@ -326,9 +330,13 @@ public void run() { private boolean flushBuffer(int bufferLength) { bufferLength = Math.min(bufferLength, this.buffer.length); - for (int i = 0; i < bufferLength && terminationFlag.running(); i++) { + int i = 0; + while (i < bufferLength && terminationFlag.running()) { try { - walks.put(this.buffer[i]); + // allow termination to occur if queue is full + if (walks.offer(this.buffer[i], 100, TimeUnit.MILLISECONDS)) { + i++; + } } catch (InterruptedException e) { Thread.currentThread().interrupt(); return false; From 6ec01a2562cfe8c96dcd0738a826ae6a9d73bf9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 6 Apr 2023 14:05:29 +0200 Subject: [PATCH 325/400] Check GDS state inside shutdown Allowing to end up in a waiting or failed state. --- .../main/java/org/neo4j/gds/utils/StringJoining.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/string-formatting/src/main/java/org/neo4j/gds/utils/StringJoining.java b/string-formatting/src/main/java/org/neo4j/gds/utils/StringJoining.java index 436066925d0..d4d1dbd12af 100644 --- a/string-formatting/src/main/java/org/neo4j/gds/utils/StringJoining.java +++ b/string-formatting/src/main/java/org/neo4j/gds/utils/StringJoining.java @@ -59,7 +59,16 @@ public static String join( CharSequence prefix, CharSequence suffix ) { - return alternatives.stream().sorted().collect(joining(delimiter, prefix, suffix)); + return join(alternatives.stream(), delimiter, prefix, suffix); + } + + public static String join( + Stream alternatives, + CharSequence delimiter, + CharSequence prefix, + CharSequence suffix + ) { + return alternatives.sorted().collect(joining(delimiter, prefix, suffix)); } public static String joinVerbose(Collection alternatives) { From 77555f7180ae3ba70668da8bab39240c4448eda1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 13 Apr 2023 16:22:21 +0200 Subject: [PATCH 326/400] Allow listening to model catalog insertions This will be used by the Shutdown to track models that were not backed up (instead of dropping the model). Also metrics can be implemented based on this --- .../neo4j/gds/core/model/ModelCatalog.java | 2 ++ .../gds/core/model/ModelCatalogListener.java | 25 +++++++++++++++++++ .../gds/core/model/OpenModelCatalog.java | 11 ++++++++ .../gds/core/model/OpenModelCatalogTest.java | 25 +++++++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalogListener.java diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java index 8cb4740c2a7..729af8ba571 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java @@ -28,6 +28,8 @@ public interface ModelCatalog { + void registerListener(ModelCatalogListener listener); + void set(Model model); Model get( diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalogListener.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalogListener.java new file mode 100644 index 00000000000..8af4661122e --- /dev/null +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalogListener.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.model; + +public interface ModelCatalogListener { + + void onInsert(Model model); +} diff --git a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java index 5529baf2e75..e5c6c587864 100644 --- a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java +++ b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; @@ -38,8 +39,16 @@ public final class OpenModelCatalog implements ModelCatalog { private final Map userCatalogs; + private final List listeners; + public OpenModelCatalog() { this.userCatalogs = new ConcurrentHashMap<>(); + this.listeners = new ArrayList<>(); + } + + @Override + public void registerListener(ModelCatalogListener listener) { + listeners.add(listener); } @Override @@ -51,6 +60,8 @@ public void set(Model model) { userCatalog.set(model); return userCatalog; }); + + listeners.forEach(listener -> listener.onInsert(model)); } @Override diff --git a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java index e41f0a96b20..016bbe56045 100644 --- a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java +++ b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.model; +import org.eclipse.collections.impl.Counter; import org.junit.jupiter.api.Test; import org.neo4j.gds.annotation.Configuration; import org.neo4j.gds.annotation.ValueClass; @@ -118,6 +119,30 @@ void shouldStoreModelsPerType() { ); } + @Test + void shouldNotifyListeners() { + var counter = new Counter(); + modelCatalog.registerListener(model -> counter.increment()); + + var model = Model.of( + "testAlgo", + GRAPH_SCHEMA, + "testTrainData", + TestTrainConfig.of(USERNAME, "testModel"), + new TestCustomInfo() + ); + + assertThat(counter.getCount()).isEqualTo(0); + + modelCatalog.set(model); + + assertThat(counter.getCount()).isEqualTo(1); + + assertThatThrownBy(() -> modelCatalog.set(model)); + // not be called if the set is not successful + assertThat(counter.getCount()).isEqualTo(1); + } + @Test void shouldThrowWhenPublishing() { modelCatalog.set(TEST_MODEL); From fe50d7aadf74b2dbd066b3552baea3b0c2fc613d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 17 Apr 2023 12:41:15 +0200 Subject: [PATCH 327/400] Fix cherry pick to 2.3 --- .../java/org/neo4j/gds/core/model/OpenModelCatalogTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java index 016bbe56045..fc5b089699c 100644 --- a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java +++ b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java @@ -129,7 +129,7 @@ void shouldNotifyListeners() { GRAPH_SCHEMA, "testTrainData", TestTrainConfig.of(USERNAME, "testModel"), - new TestCustomInfo() + Map::of ); assertThat(counter.getCount()).isEqualTo(0); From fe24791b27d2b230cc434339e9c1b362ae00e79b Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 18 Apr 2023 06:47:07 +0100 Subject: [PATCH 328/400] Update note in GDS on cluster docs --- doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index a1fb3c558e7..5c3f77520a0 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -46,7 +46,7 @@ For more information on setting up, configuring and managing a Neo4j cluster, pl ====== When working with cluster configuration you should beware https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_server.config.strict_validation.enabled[strict config validation] in Neo4j. -When configuring GDS for a Read Replica you will introduce GDS-specific configuration into `neo4j.conf` - and that is fine because with the GDS plugin installed, Neo4j will happily validate those configuration items. +When configuring GDS for a Secondary instance you will introduce GDS-specific configuration into `neo4j.conf` - and that is fine because with the GDS plugin installed, Neo4j will happily validate those configuration items. However, you might not be able to reuse that same configuration file verbatim on the core cluster members, because there you will _not_ install GDS plugin, and thus Neo4j will _not_ be able to validate the GDS-specific configuration items. And validation failure would mean Neo4j would refuse to start. From fe5d867a93ce17ba529f77ad24425a88c4932bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 20 Apr 2023 09:48:49 +0200 Subject: [PATCH 329/400] Publish database v5.7 compatibility --- build.gradle | 2 ++ settings.gradle | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/build.gradle b/build.gradle index 892e9123e2d..52b8512d36b 100644 --- a/build.gradle +++ b/build.gradle @@ -35,6 +35,7 @@ ext { project(':neo4j-kernel-adapter-5.4'), project(':neo4j-kernel-adapter-5.5'), project(':neo4j-kernel-adapter-5.6'), + project(':neo4j-kernel-adapter-5.7'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -44,6 +45,7 @@ ext { project(':storage-engine-adapter-5.4'), project(':storage-engine-adapter-5.5'), project(':storage-engine-adapter-5.6'), + project(':storage-engine-adapter-5.7'), ] ] } diff --git a/settings.gradle b/settings.gradle index 7786a5fbd66..6b6ada88d92 100644 --- a/settings.gradle +++ b/settings.gradle @@ -142,6 +142,9 @@ project(':neo4j-kernel-adapter-5.5').projectDir = file('compatibility/5.5/neo4j- include('neo4j-kernel-adapter-5.6') project(':neo4j-kernel-adapter-5.6').projectDir = file('compatibility/5.6/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.7') +project(':neo4j-kernel-adapter-5.7').projectDir = file('compatibility/5.7/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -232,6 +235,9 @@ project(':storage-engine-adapter-5.5').projectDir = file('compatibility/5.5/stor include('storage-engine-adapter-5.6') project(':storage-engine-adapter-5.6').projectDir = file('compatibility/5.6/storage-engine-adapter') +include('storage-engine-adapter-5.7') +project(':storage-engine-adapter-5.7').projectDir = file('compatibility/5.7/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') From 4ac240d73e737580180f61dd9bf23d7ec586dabe Mon Sep 17 00:00:00 2001 From: Adam Schill Collberg Date: Thu, 20 Apr 2023 14:54:03 +0200 Subject: [PATCH 330/400] Add docs for `randomSeed` parameter in RWR --- doc/modules/ROOT/pages/management-ops/projections/rwr.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc b/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc index 6044fd9af4a..fdd93828c4d 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/rwr.adoc @@ -104,6 +104,7 @@ include::partial$/algorithms/common-configuration/common-stream-stats-configurat | restartProbability | Float | 0.1 | yes | The probability that a sampling random walk restarts from one of the start nodes. | startNodes | List of Integer | A node chosen uniformly at random | yes | IDs of the initial set of nodes of the original graph from which the sampling random walks will start. | nodeLabelStratification | Boolean | false | yes | If true, preserves the node label distribution of the original graph. +| randomSeed | Integer | n/a | yes | A random seed which is used for all randomness in the computation. Requires `concurrency = 1`. |=== .Results From 14ddedc15f2c48d2889f19bd1ff71afdf62e264a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Thu, 20 Apr 2023 15:50:37 +0200 Subject: [PATCH 331/400] Update versions after 2.3.3 release --- README.adoc | 19 ++++++++++--------- .../management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.adoc b/README.adoc index 7906d6dc416..8164888cc3f 100644 --- a/README.adoc +++ b/README.adoc @@ -83,7 +83,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.2.0 |Neo4j 5.3.0 |Neo4j 5.4.0 -.7+<.^|GDS 2.3.x +.8+<.^|GDS 2.3.x |Neo4j 4.4.9 - 4.4.19 |Neo4j 5.1.0 |Neo4j 5.2.0 @@ -91,6 +91,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.4.0 |Neo4j 5.5.0 |Neo4j 5.6.0 +|Neo4j 5.7.0 |=== NOTE: Preview releases are not automatically made available in Neo4j Desktop. They need to be installed manually. @@ -134,7 +135,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.3.2 + 2.3.3 ---- @@ -146,21 +147,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.2 + 2.3.3 org.neo4j.gds algo - 2.3.2 + 2.3.3 org.neo4j.gds alpha-algo - 2.3.2 + 2.3.3 ---- @@ -172,28 +173,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.2 + 2.3.3 org.neo4j.gds proc - 2.3.2 + 2.3.3 org.neo4j.gds alpha-proc - 2.3.2 + 2.3.3 org.neo4j.gds write-services - 2.3.2 + 2.3.3 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 79ea3867dcd..adc1fdaf899 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.3" +| "2.3.4" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index ae83fd656d5..18061f66d61 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.3' + gdsVersion = '2.3.4' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index 22b877c2e52..537f719aa24 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.3.3' + gdsBaseVersion = '2.3.4' gdsAuraVersion = '17' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From 45781183ea4947d88713ce6404dbb4f5fb82519e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 21 Apr 2023 10:56:15 +0200 Subject: [PATCH 332/400] Add version mapping for scala and log4j-api --- gradle/dependencies.gradle | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 6bdc226e98c..53fe1e9c52f 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -23,7 +23,8 @@ ext { '5.4': '2.13.8', '5.5': '2.13.8', '5.6': '2.13.8', - '5.7': '2.13.10' + '5.7': '2.13.10', + '5.8': '2.13.10', ] log4js = [ @@ -34,7 +35,8 @@ ext { '5.4': '2.19.0', '5.5': '2.19.0', '5.6': '2.19.0', - '5.7': '2.20.0' + '5.7': '2.20.0', + '5.8': '2.20.0', ] ver = [ From 51eb516abf53e1a1ec80efdab3a2c61944aa1b1c Mon Sep 17 00:00:00 2001 From: yuval Date: Mon, 17 Apr 2023 14:20:19 +0200 Subject: [PATCH 333/400] Added timeout to BaseTest and BenchmarksTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The motivation is to catch tests that hang. We cannot rely on the team city timeout of 8 hours because we don't see the actual test that fails. Co-authored-by: Florentin Dörre --- test-utils/src/main/java/org/neo4j/gds/BaseTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java index b0ac81be9a3..7ce1963ba8b 100644 --- a/test-utils/src/main/java/org/neo4j/gds/BaseTest.java +++ b/test-utils/src/main/java/org/neo4j/gds/BaseTest.java @@ -22,6 +22,7 @@ import org.assertj.core.api.Assertions; import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.Timeout; import org.neo4j.gds.core.Settings; import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; @@ -38,6 +39,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -49,6 +51,7 @@ @ImpermanentDbmsExtension(configurationCallback = "configuration") @Neo4jGraphExtension +@Timeout(value = 30, unit = TimeUnit.MINUTES) public abstract class BaseTest { protected static final String DEFAULT_GRAPH_NAME = "graph"; From 501adffe1d629df6918290ec299fe5205f7133ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 17 Apr 2023 16:12:33 +0200 Subject: [PATCH 334/400] Fix chained cypher queries locking the test For some reason SHOW index + DROP index does not work with dev --- .../gds/core/loading/NodeLabelIndexTest.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/core/src/test/java/org/neo4j/gds/core/loading/NodeLabelIndexTest.java b/core/src/test/java/org/neo4j/gds/core/loading/NodeLabelIndexTest.java index 03fa1be388f..372667ee5c7 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/NodeLabelIndexTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/NodeLabelIndexTest.java @@ -27,6 +27,9 @@ import org.neo4j.gds.extension.Neo4jGraph; import org.neo4j.gds.extension.Neo4jGraphExtension; +import java.util.ArrayList; +import java.util.List; + import static org.assertj.core.api.Assertions.assertThat; import static org.neo4j.gds.TestSupport.assertGraphEquals; import static org.neo4j.gds.TestSupport.fromGdl; @@ -39,17 +42,26 @@ public class NodeLabelIndexTest extends BaseTest { @Test void shouldLoadWithoutNodeLabelIndex() { - runQueryWithResultConsumer( - "SHOW INDEXES WHERE entityType = 'NODE'", + List nodeIndices = runQuery( + "SHOW INDEXES " + + " YIELD name, entityType " + + " WHERE entityType = 'NODE'" + + " RETURN name", result -> { - assertThat(result.hasNext()).isTrue(); - runQueryWithResultConsumer( - "DROP INDEX " + result.next().get("name"), - innerResult -> assertThat(innerResult.resultAsString()).contains("Indexes removed: 1") - ); + var indices = new ArrayList(); + while (result.hasNext()) { + indices.add((String) result.next().get("name")); + } + return indices; } ); + nodeIndices.forEach(index -> runQuery( + "DROP INDEX " + index, + result -> assertThat(result.resultAsString()).contains("Indexes removed: 1") + )); + + var log = Neo4jProxy.testLog();; var graph = new StoreLoaderBuilder() .databaseService(db) From 974c51bc3b68503c46da9535f42ece1df023fa76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 20 Apr 2023 18:03:03 +0200 Subject: [PATCH 335/400] Handle stored models in backup/shutdown_new and restore This also underlines the problem of the old shutdown dropping resources. We also always load the stored models for convenience (and avoid unexpectedly unloaded models inside Aura) --- .../org/neo4j/gds/core/model/ModelHash.java | 51 +++++++++++++++++++ .../neo4j/gds/core/model/ModelMetaData.java | 8 +++ 2 files changed, 59 insertions(+) create mode 100644 model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelHash.java diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelHash.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelHash.java new file mode 100644 index 00000000000..eaf43967ef4 --- /dev/null +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelHash.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.model; + +import org.neo4j.gds.annotation.ValueClass; +import org.neo4j.gds.model.ModelConfig; + +import java.time.ZonedDateTime; + +@ValueClass +public +interface ModelHash { + String user(); + + String name(); + + ZonedDateTime timestamp(); + + static ModelHash of(ModelMetaData modelMetaData) { + return ImmutableModelHash.of( + modelMetaData.creator(), + modelMetaData.name(), + modelMetaData.creationTime() + ); + } + + static ModelHash of(Model model) { + return ImmutableModelHash.of( + model.creator(), + model.name(), + model.creationTime() + ); + } +} diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelMetaData.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelMetaData.java index a304d31bb7b..1336b8ccda0 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelMetaData.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelMetaData.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.model; +import org.immutables.value.Value; import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.schema.GraphSchema; import org.neo4j.gds.config.ToMapConvertible; @@ -27,6 +28,8 @@ import java.time.ZonedDateTime; import java.util.List; +import static org.neo4j.gds.core.model.Model.ALL_USERS; + @ValueClass public interface ModelMetaData { @@ -45,4 +48,9 @@ public interface ModelMetaData Date: Thu, 20 Apr 2023 17:52:19 +0200 Subject: [PATCH 336/400] Doc restructure around CELF --- .../src/test/java/org/neo4j/gds/doc/CELFDocTest.java | 2 +- .../java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java | 2 +- doc/modules/ROOT/content-nav.adoc | 3 +-- .../algorithms/{influence-maximization => }/celf.adoc | 3 +++ doc/modules/ROOT/pages/algorithms/centrality.adoc | 2 +- .../ROOT/pages/algorithms/influence-maximization.adoc | 11 ----------- .../operations-reference/algorithm-references.adoc | 2 +- 7 files changed, 8 insertions(+), 17 deletions(-) rename doc/modules/ROOT/pages/algorithms/{influence-maximization => }/celf.adoc (99%) delete mode 100644 doc/modules/ROOT/pages/algorithms/influence-maximization.adoc diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/CELFDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/CELFDocTest.java index ad0b1620170..5ac7bd6cbd5 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/CELFDocTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/CELFDocTest.java @@ -50,6 +50,6 @@ protected List> procedures() { @Override protected String adocFile() { - return "pages/algorithms/influence-maximization/celf.adoc"; + return "pages/algorithms/celf.adoc"; } } diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java index 11cc2253e20..e1e097effdf 100644 --- a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/CELFSyntaxTest.java @@ -39,6 +39,6 @@ protected Iterable syntaxModes() { @Override protected String adocFile() { - return "pages/algorithms/influence-maximization/celf.adoc"; + return "pages/algorithms/celf.adoc"; } } diff --git a/doc/modules/ROOT/content-nav.adoc b/doc/modules/ROOT/content-nav.adoc index 713832e484b..71191a6c002 100644 --- a/doc/modules/ROOT/content-nav.adoc +++ b/doc/modules/ROOT/content-nav.adoc @@ -49,8 +49,7 @@ *** xref:algorithms/closeness-centrality.adoc[] *** xref:algorithms/harmonic-centrality.adoc[] *** xref:algorithms/hits.adoc[] -*** xref:algorithms/influence-maximization.adoc[] -**** xref:algorithms/influence-maximization/celf.adoc[] +*** xref:algorithms/celf.adoc[] ** xref:algorithms/community.adoc[] *** xref:algorithms/louvain.adoc[] *** xref:algorithms/label-propagation.adoc[] diff --git a/doc/modules/ROOT/pages/algorithms/influence-maximization/celf.adoc b/doc/modules/ROOT/pages/algorithms/celf.adoc similarity index 99% rename from doc/modules/ROOT/pages/algorithms/influence-maximization/celf.adoc rename to doc/modules/ROOT/pages/algorithms/celf.adoc index d3aa3b1e0d1..f9b0cdac1de 100644 --- a/doc/modules/ROOT/pages/algorithms/influence-maximization/celf.adoc +++ b/doc/modules/ROOT/pages/algorithms/celf.adoc @@ -1,3 +1,6 @@ +:page-aliases: algorithms/influence-maximization/celf.adoc +:page-aliases: algorithms/influence-maximization/ + [[algorithms-celf]] [.beta] = CELF diff --git a/doc/modules/ROOT/pages/algorithms/centrality.adoc b/doc/modules/ROOT/pages/algorithms/centrality.adoc index b9d6f5541b7..4ecb475265a 100644 --- a/doc/modules/ROOT/pages/algorithms/centrality.adoc +++ b/doc/modules/ROOT/pages/algorithms/centrality.adoc @@ -17,4 +17,4 @@ The Neo4j GDS library includes the following centrality algorithms, grouped by q * Alpha ** xref:algorithms/harmonic-centrality.adoc[Harmonic Centrality] ** xref:algorithms/hits.adoc[HITS] -** xref:algorithms/influence-maximization.adoc[Influence Maximization] +** xref:algorithms/celf.adoc[CELF] diff --git a/doc/modules/ROOT/pages/algorithms/influence-maximization.adoc b/doc/modules/ROOT/pages/algorithms/influence-maximization.adoc deleted file mode 100644 index 798127310d6..00000000000 --- a/doc/modules/ROOT/pages/algorithms/influence-maximization.adoc +++ /dev/null @@ -1,11 +0,0 @@ -[[algorithms-influence-maximization]] -= Influence Maximization -:description: This chapter provides explanations and examples for each of the influence maximization algorithms in the Neo4j Graph Data Science library. - - -The objective of influence maximization is to find a small subset of k nodes from a network in order to achieve maximization to the total number of nodes influenced by these k nodes. -The Neo4j GDS library includes the following alpha influence maximization algorithms: - -* Beta -** xref:algorithms/influence-maximization/celf.adoc[CELF] - diff --git a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc index aefb224da8f..74d6c20a128 100644 --- a/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc +++ b/doc/modules/ROOT/pages/operations-reference/algorithm-references.adoc @@ -239,7 +239,7 @@ | `gds.beta.node2vec.stream.estimate` | `gds.beta.node2vec.write` | `gds.beta.node2vec.write.estimate` -.8+<.^| xref:algorithms/influence-maximization/celf.adoc[Influence Maximization - CELF] +.8+<.^| xref:algorithms/celf.adoc[Influence Maximization - CELF] | `gds.beta.influenceMaximization.celf.mutate` | `gds.beta.influenceMaximization.celf.mutate.estimate` | `gds.beta.influenceMaximization.celf.stats` From 5d2185cd574d787b29e8da648ac6f6932c27f646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 18 Apr 2023 17:18:19 +0200 Subject: [PATCH 337/400] Register inserts during shutdown instead of dropping backed up graphs This will allow us to not interrupt existing work flows but actually wait for a good time to shutdown. Also the catalog listener can be used for metrics in the future. --- .../gds/core/loading/GraphStoreCatalog.java | 14 +++++++++++ .../loading/GraphStoreCatalogListener.java | 25 +++++++++++++++++++ .../core/loading/GraphStoreCatalogTest.java | 23 +++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java diff --git a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java index 306a55710de..500a212fc2c 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java @@ -27,6 +27,8 @@ import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.utils.StringJoining; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -41,8 +43,18 @@ public final class GraphStoreCatalog { private static final ConcurrentHashMap userCatalogs = new ConcurrentHashMap<>(); + private static final List listeners = new ArrayList<>(); + private GraphStoreCatalog() { } + public static void registerListener(GraphStoreCatalogListener listener) { + listeners.add(listener); + } + + public static void unregisterListener(GraphStoreCatalogListener listener) { + listeners.remove(listener); + } + public static GraphStoreWithConfig get(CatalogRequest request, String graphName) { var userCatalogKey = UserCatalog.UserCatalogKey.of(request.databaseName(), graphName); var ownCatalog = getUserCatalog(request.username()); @@ -161,6 +173,8 @@ private static void set(GraphProjectConfig config, GraphStore graphStore, boolea ); return userCatalog; }); + + listeners.forEach(listener -> listener.onProject(config.username(), config.graphName())); } public static boolean exists(String username, String databaseName, String graphName) { diff --git a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java new file mode 100644 index 00000000000..da3e3025e2a --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.loading; + +public interface GraphStoreCatalogListener { + + void onProject(String user, String graphName); +} diff --git a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java index e4f6fc64510..8b10e2fd6b3 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.gds.core.loading; +import org.apache.commons.lang3.mutable.MutableInt; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.neo4j.gds.api.DatabaseId; @@ -414,6 +415,28 @@ void multipleDatabaseIds() { assertFalse(GraphStoreCatalog.exists(USER_NAME, databaseId1, "graph0")); } + @Test + void callListeners() { + var listenerCalls = new MutableInt(0); + + var listener = new GraphStoreCatalogListener() { + @Override + public void onProject(String user, String graphName) { + listenerCalls.increment(); + } + }; + + GraphStoreCatalog.registerListener(listener); + assertThat(listenerCalls.intValue()).isZero(); + + GraphStoreCatalog.set(CONFIG, graphStore); + assertThat(listenerCalls.intValue()).isEqualTo(1); + + GraphStoreCatalog.unregisterListener(listener); + GraphStoreCatalog.set(GraphProjectFromStoreConfig.emptyWithName("bob", GRAPH_NAME), graphStore); + assertThat(listenerCalls.intValue()).isEqualTo(1); + } + @Test void shouldThrowOnMissingGraph() { var dummyDatabaseId = DatabaseId.from("mydatabase"); From 607e13b015b17c7c773ff7e127a96fb921f9e4a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 24 Apr 2023 10:54:04 +0200 Subject: [PATCH 338/400] Pass log to GraphStoreCatalog on dbms startup The log is used to warn if a listener fails --- .../org/neo4j/gds/core/GdsLogExtension.java | 50 +++++++++++++++++++ .../gds/core/loading/GraphStoreCatalog.java | 23 ++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/GdsLogExtension.java diff --git a/core/src/main/java/org/neo4j/gds/core/GdsLogExtension.java b/core/src/main/java/org/neo4j/gds/core/GdsLogExtension.java new file mode 100644 index 00000000000..72de2c3f8e2 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/GdsLogExtension.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxy; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.kernel.extension.ExtensionFactory; +import org.neo4j.kernel.extension.ExtensionType; +import org.neo4j.kernel.extension.context.ExtensionContext; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.logging.internal.LogService; + +@ServiceProvider +public class GdsLogExtension extends ExtensionFactory { + + public GdsLogExtension() { + super(ExtensionType.GLOBAL, "gds.log"); + } + + @Override + public Lifecycle newInstance(ExtensionContext context, Dependencies dependencies) { + return LifecycleAdapter.onInit(() -> { + var log = Neo4jProxy.getUserLog(dependencies.logService(), GdsLogExtension.class); + GraphStoreCatalog.setLog(log); + }); + } + + interface Dependencies { + LogService logService(); + } +} diff --git a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java index 500a212fc2c..5dce7fa434d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java @@ -26,9 +26,11 @@ import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.config.GraphProjectConfig; import org.neo4j.gds.utils.StringJoining; +import org.neo4j.logging.Log; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -45,6 +47,10 @@ public final class GraphStoreCatalog { private static final List listeners = new ArrayList<>(); + // as we want to use the Neo4j log if possible and the catalog is a static instance, + // we make the log injectable + private static Optional log = Optional.empty(); + private GraphStoreCatalog() { } public static void registerListener(GraphStoreCatalogListener listener) { @@ -55,6 +61,10 @@ public static void unregisterListener(GraphStoreCatalogListener listener) { listeners.remove(listener); } + public static void setLog(Log log) { + GraphStoreCatalog.log = Optional.of(log); + } + public static GraphStoreWithConfig get(CatalogRequest request, String graphName) { var userCatalogKey = UserCatalog.UserCatalogKey.of(request.databaseName(), graphName); var ownCatalog = getUserCatalog(request.username()); @@ -174,7 +184,18 @@ private static void set(GraphProjectConfig config, GraphStore graphStore, boolea return userCatalog; }); - listeners.forEach(listener -> listener.onProject(config.username(), config.graphName())); + listeners.forEach(listener -> { + try { + listener.onProject(config.username(), config.graphName()); + } catch (Exception e) { + log.ifPresent(l -> l.warn(String.format( + Locale.US, + "Could not call listener %s on setting the graph %s", + listener, + config.graphName() + ), e)); + } + }); } public static boolean exists(String username, String databaseName, String graphName) { From 7d674f79dd05b6f3ec82ab2e8c9ec84ed01942da Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 25 Apr 2023 04:41:59 +0100 Subject: [PATCH 339/400] Clarify GDS on Neo4j cluster in 5.x setup --- .../ROOT/pages/production-deployment/neo4j-cluster.adoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index 5c3f77520a0..dc762a7a572 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -23,8 +23,9 @@ Since GDS performs large computations with the full resources of the system it i We make use of a _Secondary_ instance to deploy the GDS library and process analytical workloads. Calls to GDS `write` procedures are internally directed via _server-side routing_ to the cluster instance that is a `Writer` for the database we work on. -Neo4j 5.x supports different databases on the same machine to act as `Primary` or `Secondary` members of the cluster, we *do not* recommend mixed setup. -GDS should be installed on a machine that is not serving transactional load and does not participate in `Leader` elections. +Neo4j 5.x supports different databases on the same cluster instance to act as `Primary` or `Secondary` members of the cluster. +In order for GDS to work correctly, all databases on the instance it is installed have to be `Secondary`, including the `system` database (see https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_server.cluster.system_database_mode[server.cluster.system_database_mode] and https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_initial.server.mode_constraint[initial.server.mode_constraint]). +GDS has compute-intensive OLAP workloads that may disrupt the cluster operations and we recommend GDS to be installed on an instance that is not serving transactional load and does not participate in `Leader` elections. [NOTE] ====== From 33da470226427bdc8268100d19f9d353f295d166 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 25 Apr 2023 10:58:07 +0100 Subject: [PATCH 340/400] Update public/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc Co-authored-by: Lasse Westh-Nielsen --- doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index dc762a7a572..f7b5b47933b 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -24,7 +24,7 @@ We make use of a _Secondary_ instance to deploy the GDS library and process anal Calls to GDS `write` procedures are internally directed via _server-side routing_ to the cluster instance that is a `Writer` for the database we work on. Neo4j 5.x supports different databases on the same cluster instance to act as `Primary` or `Secondary` members of the cluster. -In order for GDS to work correctly, all databases on the instance it is installed have to be `Secondary`, including the `system` database (see https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_server.cluster.system_database_mode[server.cluster.system_database_mode] and https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_initial.server.mode_constraint[initial.server.mode_constraint]). +In order for GDS to function, all databases on the instance it is installed have to be `Secondary`, including the `system` database (see https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_server.cluster.system_database_mode[server.cluster.system_database_mode] and https://neo4j.com/docs/operations-manual/current/reference/configuration-settings/#config_initial.server.mode_constraint[initial.server.mode_constraint]). GDS has compute-intensive OLAP workloads that may disrupt the cluster operations and we recommend GDS to be installed on an instance that is not serving transactional load and does not participate in `Leader` elections. [NOTE] From de316e0e6c419fe6bfbb5c085b56c708a4dc68ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 24 Apr 2023 16:21:02 +0200 Subject: [PATCH 341/400] Register base metric extension and expose first metric --- .../main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java | 2 +- .../org/neo4j/gds/core/loading/GraphStoreCatalogListener.java | 2 +- .../java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java index 5dce7fa434d..754a111d0ac 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java @@ -186,7 +186,7 @@ private static void set(GraphProjectConfig config, GraphStore graphStore, boolea listeners.forEach(listener -> { try { - listener.onProject(config.username(), config.graphName()); + listener.onProject(config.username(), graphStore.databaseId().databaseName(), config.graphName()); } catch (Exception e) { log.ifPresent(l -> l.warn(String.format( Locale.US, diff --git a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java index da3e3025e2a..3b28be35ab0 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalogListener.java @@ -21,5 +21,5 @@ public interface GraphStoreCatalogListener { - void onProject(String user, String graphName); + void onProject(String user, String database, String graphName); } diff --git a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java index 8b10e2fd6b3..18e9e7ad556 100644 --- a/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java +++ b/core/src/test/java/org/neo4j/gds/core/loading/GraphStoreCatalogTest.java @@ -421,7 +421,7 @@ void callListeners() { var listener = new GraphStoreCatalogListener() { @Override - public void onProject(String user, String graphName) { + public void onProject(String user, String database, String graphName) { listenerCalls.increment(); } }; From 068ec3dda798766b88d77d65393a2fcf4838ecbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 24 Apr 2023 16:23:56 +0200 Subject: [PATCH 342/400] Change listeners in Catalog from List to Set makes more sense as the same listener should not be added twice. Also faster remove op. --- .../java/org/neo4j/gds/core/loading/GraphStoreCatalog.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java index 754a111d0ac..00c11f4493d 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java @@ -28,12 +28,12 @@ import org.neo4j.gds.utils.StringJoining; import org.neo4j.logging.Log; -import java.util.ArrayList; -import java.util.List; +import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -45,7 +45,7 @@ public final class GraphStoreCatalog { private static final ConcurrentHashMap userCatalogs = new ConcurrentHashMap<>(); - private static final List listeners = new ArrayList<>(); + private static final Set listeners = new HashSet<>(); // as we want to use the Neo4j log if possible and the catalog is a static instance, // we make the log injectable From 8aaf93ceb9be352a5aaa06caffa93774c4de21e7 Mon Sep 17 00:00:00 2001 From: Jacob Sznajdman Date: Wed, 26 Apr 2023 12:56:39 +0200 Subject: [PATCH 343/400] Fix sampling bugs related to idmaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre Co-Authored-By: Adam Schill Collberg --- .../ml/negativeSampling/NegativeSampler.java | 8 +- .../UserInputNegativeSampler.java | 7 +- .../ml/splitting/DirectedEdgeSplitter.java | 11 ++- .../neo4j/gds/ml/splitting/EdgeSplitter.java | 11 ++- .../gds/ml/splitting/SplitRelationships.java | 11 ++- .../ml/splitting/UndirectedEdgeSplitter.java | 5 +- .../splitting/DirectedEdgeSplitterTest.java | 10 ++- .../splitting/UndirectedEdgeSplitterTest.java | 25 ++++-- .../UserInputNegativeSamplerTest.java | 13 ++- .../LinkPredictionRelationshipSampler.java | 9 ++- ...LinkPredictionRelationshipSamplerTest.java | 79 ++++++++++++++++++- 11 files changed, 163 insertions(+), 26 deletions(-) diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/NegativeSampler.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/NegativeSampler.java index 0f222872440..40a0f92f6cd 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/NegativeSampler.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/NegativeSampler.java @@ -27,6 +27,7 @@ import org.neo4j.gds.core.loading.construction.RelationshipsBuilder; import java.util.Collection; +import java.util.List; import java.util.Optional; public interface NegativeSampler { @@ -36,6 +37,7 @@ public interface NegativeSampler { static NegativeSampler of( GraphStore graphStore, Graph graph, + Collection sourceAndTargetNodeLabels, Optional negativeRelationshipType, double negativeSamplingRatio, long testPositiveCount, @@ -47,7 +49,11 @@ static NegativeSampler of( Optional randomSeed ) { if (negativeRelationshipType.isPresent()) { - Graph negativeExampleGraph = graphStore.getGraph(RelationshipType.of(negativeRelationshipType.orElseThrow())); + Graph negativeExampleGraph = graphStore.getGraph( + sourceAndTargetNodeLabels, + List.of(RelationshipType.of(negativeRelationshipType.orElseThrow())), + Optional.empty() + ); double testTrainFraction = testPositiveCount / (double) (testPositiveCount + trainPositiveCount); return new UserInputNegativeSampler( diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/UserInputNegativeSampler.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/UserInputNegativeSampler.java index b19af6fc804..7a3db98a57f 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/UserInputNegativeSampler.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/negativeSampling/UserInputNegativeSampler.java @@ -64,13 +64,16 @@ public void produceNegativeSamples( negativeExampleGraph.forEachNode(nodeId -> { negativeExampleGraph.forEachRelationship(nodeId, (s, t) -> { + // as we are adding the relationships to the GraphStore we need to operate over the rootNodeIds + long rootS = negativeExampleGraph.toRootNodeId(s); + long rootT = negativeExampleGraph.toRootNodeId(t); if (s < t) { if (sample(testRelationshipsToAdd.doubleValue()/(testRelationshipsToAdd.doubleValue() + trainRelationshipsToAdd.doubleValue()))) { testRelationshipsToAdd.decrement(); - testSetBuilder.add(s, t, NEGATIVE); + testSetBuilder.addFromInternal(rootS, rootT, NEGATIVE); } else { trainRelationshipsToAdd.decrement(); - trainSetBuilder.add(s, t, NEGATIVE); + trainSetBuilder.addFromInternal(rootS, rootT, NEGATIVE); } } return true; diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java index 44b46d421ec..3c86dff2ba4 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitter.java @@ -36,13 +36,22 @@ public class DirectedEdgeSplitter extends EdgeSplitter { public DirectedEdgeSplitter( Optional maybeSeed, + IdMap rootNodes, IdMap sourceLabels, IdMap targetLabels, RelationshipType selectedRelationshipType, RelationshipType remainingRelationshipType, int concurrency ) { - super(maybeSeed, sourceLabels, targetLabels, selectedRelationshipType, remainingRelationshipType, concurrency); + super( + maybeSeed, + rootNodes, + sourceLabels, + targetLabels, + selectedRelationshipType, + remainingRelationshipType, + concurrency + ); } @Override diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java index 8e6d34ece1e..db52570ae92 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/EdgeSplitter.java @@ -47,17 +47,20 @@ public abstract class EdgeSplitter { protected final IdMap sourceNodes; protected final IdMap targetNodes; + protected final IdMap rootNodes; protected int concurrency; EdgeSplitter( Optional maybeSeed, + IdMap rootNodes, IdMap sourceNodes, IdMap targetNodes, RelationshipType selectedRelationshipType, RelationshipType remainingRelationshipType, int concurrency ) { + this.rootNodes = rootNodes; this.selectedRelationshipType = selectedRelationshipType; this.remainingRelationshipType = remainingRelationshipType; this.rng = new Random(); @@ -78,7 +81,7 @@ public SplitResult splitPositiveExamples( LongLongPredicate isValidNodePair = (s, t) -> isValidSourceNode.apply(s) && isValidTargetNode.apply(t); RelationshipsBuilder selectedRelsBuilder = newRelationshipsBuilder( - graph, + rootNodes, selectedRelationshipType, Direction.DIRECTED, Optional.of(EdgeSplitter.RELATIONSHIP_PROPERTY) @@ -89,7 +92,7 @@ public SplitResult splitPositiveExamples( RelationshipsBuilder remainingRelsBuilder; RelationshipWithPropertyConsumer remainingRelsConsumer; - remainingRelsBuilder = newRelationshipsBuilder(graph, remainingRelationshipType, remainingRelDirection, remainingRelPropertyKey); + remainingRelsBuilder = newRelationshipsBuilder(rootNodes, remainingRelationshipType, remainingRelDirection, remainingRelPropertyKey); remainingRelsConsumer = (s, t, w) -> { remainingRelsBuilder.addFromInternal(graph.toRootNodeId(s), graph.toRootNodeId(t), w); return true; @@ -153,7 +156,7 @@ protected long samplesPerNode(long maxSamples, double remainingSamples, long rem } private static RelationshipsBuilder newRelationshipsBuilder( - Graph graph, + IdMap rootNodes, RelationshipType relationshipType, Direction direction, Optional propertyKey @@ -161,7 +164,7 @@ private static RelationshipsBuilder newRelationshipsBuilder( return GraphFactory.initRelationshipsBuilder() .relationshipType(relationshipType) .aggregation(Aggregation.SINGLE) - .nodes(graph) + .nodes(rootNodes) .orientation(direction.toOrientation()) .addAllPropertyConfigs(propertyKey .map(key -> List.of(GraphFactory.PropertyConfig.of(key, Aggregation.SINGLE, DefaultValue.forDouble()))) diff --git a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java index f09e8e3e5df..86563c863c3 100644 --- a/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java +++ b/ml/ml-algo/src/main/java/org/neo4j/gds/ml/splitting/SplitRelationships.java @@ -40,14 +40,19 @@ public final class SplitRelationships extends Algorithm maybeSeed, + IdMap rootNodes, IdMap sourceNodes, IdMap targetNodes, RelationshipType selectedRelationshipType, RelationshipType remainingRelationshipType, int concurrency ) { - super(maybeSeed, sourceNodes, targetNodes, selectedRelationshipType, remainingRelationshipType, concurrency); + super(maybeSeed, + rootNodes, + sourceNodes, targetNodes, selectedRelationshipType, remainingRelationshipType, concurrency); } @Override diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java index 025b48c97ed..2a4340c5f56 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java @@ -131,6 +131,7 @@ void splitSkewedGraph() { Optional.of(-1L), skewedGraphStore.nodes(), skewedGraphStore.nodes(), + skewedGraphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -150,6 +151,7 @@ void splitMultiGraph() { Optional.of(-1L), multiGraphStore.nodes(), multiGraphStore.nodes(), + multiGraphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -170,6 +172,7 @@ void split() { Optional.of(-1L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -207,6 +210,7 @@ void negativeEdgesShouldNotOverlapMasterGraph() { .generate(); var splitter = new DirectedEdgeSplitter(Optional.of(42L), + huuuuugeDenseGraph, huuuuugeDenseGraph, huuuuugeDenseGraph, RelationshipType.of("SELECTED"), @@ -241,6 +245,7 @@ void negativeEdgeSampling() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -261,6 +266,7 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { Collection targetNodeLabels = List.of(NodeLabel.of("C"), NodeLabel.of("D")); var splitter = new DirectedEdgeSplitter( Optional.of(1337L), + multiLabelGraphStore.nodes(), multiLabelGraphStore.getGraph(sourceNodeLabels), multiLabelGraphStore.getGraph(targetNodeLabels), RelationshipType.of("SELECTED"), @@ -280,7 +286,7 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { var selectedRelationships = result.selectedRels().build(); assertThat(selectedRelationships.topology()).satisfies(topology -> { - assertRelSamplingProperties(selectedRelationships, multiLabelGraph); + assertRelSamplingProperties(selectedRelationships, multiLabelGraphStore); assertThat(topology.elementCount()).isEqualTo(1); assertFalse(topology.isMultiGraph()); }); @@ -295,6 +301,7 @@ void samplesWithinBounds() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -310,6 +317,7 @@ void shouldPreserveRelationshipWeights() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java index 690731bcad0..2953286b5d6 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UndirectedEdgeSplitterTest.java @@ -97,6 +97,7 @@ void split() { Optional.of(1337L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -127,6 +128,7 @@ void splitMultiGraph() { Optional.of(-1L), multiGraphStore.nodes(), multiGraphStore.nodes(), + multiGraphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -157,6 +159,7 @@ void negativeEdgesShouldNotOverlapMasterGraph() { Optional.of(42L), huuuuugeDenseGraph, huuuuugeDenseGraph, + huuuuugeDenseGraph, RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -198,29 +201,31 @@ void shouldProduceDeterministicResult() { var splitResult1 = new UndirectedEdgeSplitter( Optional.of(12L), - graphStore.nodes(), - graphStore.nodes(), + graph.idMap(), + graph.idMap(), + graph.idMap(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var splitResult2 = new UndirectedEdgeSplitter( Optional.of(12L), - graphStore.nodes(), - graphStore.nodes(), + graph.idMap(), + graph.idMap(), + graph.idMap(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 ).splitPositiveExamples(graph, 0.5, Optional.empty()); var remainingAreEqual = relationshipsAreEqual( - graph, + graph.idMap(), splitResult1.remainingRels().build(), splitResult2.remainingRels().build() ); assertTrue(remainingAreEqual); var holdoutAreEqual = relationshipsAreEqual( - graph, + graph.idMap(), splitResult1.selectedRels().build(), splitResult2.selectedRels().build() ); @@ -244,6 +249,7 @@ void shouldProduceNonDeterministicResult() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -252,6 +258,7 @@ void shouldProduceNonDeterministicResult() { Optional.of(117L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -277,6 +284,7 @@ void negativeEdgeSampling() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -299,6 +307,7 @@ void splitWithFilteringWithSameSourceTargetLabels() { Optional.of(1337L), graphStore.getGraph(NodeLabel.of("A")), graphStore.getGraph(NodeLabel.of("A")), + graphStore.getGraph(NodeLabel.of("A")), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -332,6 +341,7 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { Collection targetNodeLabels = List.of(NodeLabel.of("C"), NodeLabel.of("D")); var splitter = new UndirectedEdgeSplitter( Optional.of(1337L), + multiLabelGraphStore.nodes(), multiLabelGraphStore.getGraph(sourceNodeLabels), multiLabelGraphStore.getGraph(targetNodeLabels), RelationshipType.of("SELECTED"), @@ -367,6 +377,7 @@ void samplesWithinBounds() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -382,6 +393,7 @@ void shouldPreserveRelationshipWeights() { Optional.of(42L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 @@ -411,6 +423,7 @@ void zeroNegativeSamples() { Optional.of(1337L), graphStore.nodes(), graphStore.nodes(), + graphStore.nodes(), RelationshipType.of("SELECTED"), RelationshipType.of("REMAINING"), 4 diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java index 41edc76800b..de0e7e62550 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/UserInputNegativeSamplerTest.java @@ -31,6 +31,7 @@ import org.neo4j.gds.core.loading.construction.RelationshipsBuilderBuilder; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.ml.negativeSampling.UserInputNegativeSampler; @@ -40,11 +41,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; import static org.neo4j.gds.ml.negativeSampling.NegativeSampler.NEGATIVE; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; @GdlExtension class UserInputNegativeSamplerTest { - @GdlGraph(orientation = Orientation.UNDIRECTED) + @GdlGraph(orientation = Orientation.UNDIRECTED, idOffset = 5000) static String gdl = "(a1:A), " + "(a2:A), " + @@ -61,6 +63,9 @@ class UserInputNegativeSamplerTest { @Inject Graph graph; + @Inject + IdFunction idFunction; + @Test void generateNegativeSamples() { @@ -120,9 +125,11 @@ void shouldValidateNegativeExamplesRespectNodeLabels() { List.of(NodeLabel.of("A")) )) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("There is a relationship of negativeRelationshipType between nodes 0 and 1. " + + .hasMessageContaining(formatWithLocale("There is a relationship of negativeRelationshipType between nodes %d and %d. " + "The nodes have types [NodeLabel{name='A'}] and [NodeLabel{name='A'}]. " + - "However, they need to be between [NodeLabel{name='B'}] and [NodeLabel{name='A'}]." + "However, they need to be between [NodeLabel{name='B'}] and [NodeLabel{name='A'}].", + idFunction.of("a1"), idFunction.of("a2") + ) ); } diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java index c51c61169b7..fcb42a5c8fd 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSampler.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.NotNull; import org.neo4j.gds.ElementProjection; +import org.neo4j.gds.NodeLabel; import org.neo4j.gds.RelationshipType; import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; @@ -41,6 +42,7 @@ import org.neo4j.gds.ml.splitting.EdgeSplitter; import org.neo4j.gds.ml.splitting.UndirectedEdgeSplitter; +import java.util.Collection; import java.util.List; import java.util.Optional; @@ -102,8 +104,9 @@ public void splitAndSampleRelationships( var targetLabels = ElementTypeValidator.resolve(graphStore, List.of(trainConfig.targetNodeLabel())); IdMap sourceNodes = graphStore.getGraph(sourceLabels); IdMap targetNodes = graphStore.getGraph(targetLabels); + Collection sourceAndTargetNodeLabels = trainConfig.nodeLabelIdentifiers(graphStore); var graph = graphStore.getGraph( - trainConfig.nodeLabelIdentifiers(graphStore), + sourceAndTargetNodeLabels, trainConfig.internalRelationshipTypes(graphStore), relationshipWeightProperty); @@ -121,7 +124,7 @@ public void splitAndSampleRelationships( ); // 2. Split test-complement into (labeled) train and feature-input. var testComplementGraph = graphStore.getGraph( - trainConfig.nodeLabelIdentifiers(graphStore), + sourceAndTargetNodeLabels, List.of(splitConfig.testComplementRelationshipType()), relationshipWeightProperty ); @@ -141,6 +144,7 @@ public void splitAndSampleRelationships( NegativeSampler negativeSampler = NegativeSampler.of( graphStore, graph, + sourceAndTargetNodeLabels, splitConfig.negativeRelationshipType(), splitConfig.negativeSamplingRatio(), testSplitResult.selectedRelCount(), @@ -180,6 +184,7 @@ private EdgeSplitter.SplitResult split( } var splitter = new UndirectedEdgeSplitter( trainConfig.randomSeed(), + graphStore.nodes(), sourceNodes, targetNodes, selectedRelType, diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java index 6bca527a1da..0dd392bc1c8 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java @@ -24,6 +24,7 @@ import org.neo4j.gds.InspectableTestProgressTracker; import org.neo4j.gds.Orientation; import org.neo4j.gds.RelationshipType; +import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.schema.ElementSchemaEntry; import org.neo4j.gds.assertj.Extractors; @@ -36,13 +37,17 @@ import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; import org.neo4j.gds.extension.GdlExtension; import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.IdFunction; import org.neo4j.gds.extension.Inject; import org.neo4j.gds.ml.pipeline.linkPipeline.LinkPredictionSplitConfigImpl; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -54,16 +59,43 @@ @GdlExtension class LinkPredictionRelationshipSamplerTest { - @GdlGraph(orientation = Orientation.UNDIRECTED) + @GdlGraph(orientation = Orientation.UNDIRECTED, idOffset = 1337) private static final String GRAPH = "CREATE " + + "(x1:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(x2:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(x3:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(x4:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(x5:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(x6:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(x7:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(x8:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(x9:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y1:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y2:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y3:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y4:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y5:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y6:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y7:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y8:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(y9:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(z1:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(z3:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(z5:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(z6:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(z7:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(z8:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + "(z9:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + "(a:N {scalar: 0, array: [-1.0, -2.0, 1.0, 1.0, 3.0]}), " + + "(z4:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + "(b:N {scalar: 4, array: [2.0, 1.0, -2.0, 2.0, 1.0]}), " + "(c:N {scalar: 0, array: [-3.0, 4.0, 3.0, 3.0, 2.0]}), " + "(d:N {scalar: 3, array: [1.0, 3.0, 1.0, -1.0, -1.0]}), " + "(e:N {scalar: 1, array: [-2.0, 1.0, 2.0, 1.0, -1.0]}), " + "(f:N {scalar: 0, array: [-1.0, -3.0, 1.0, 2.0, 2.0]}), " + "(g:N {scalar: 1, array: [3.0, 1.0, -3.0, 3.0, 1.0]}), " + + "(z2:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + "(h:N {scalar: 3, array: [-1.0, 3.0, 2.0, 1.0, -3.0]}), " + "(i:N {scalar: 3, array: [4.0, 1.0, 1.0, 2.0, 1.0]}), " + "(j:N {scalar: 4, array: [1.0, -4.0, 2.0, -2.0, 2.0]}), " + @@ -96,6 +128,9 @@ class LinkPredictionRelationshipSamplerTest { @Inject GraphStore graphStore; + @Inject + IdFunction idFunction; + @GdlGraph(graphNamePrefix = "multi", orientation = Orientation.UNDIRECTED) private static final String MULTI_GRAPH = "CREATE " + @@ -363,7 +398,7 @@ void splitWithSpecifiedNegativeRelationships() { .negativeRelationshipType("NEGATIVE") // 3 total .build(); - var trainConfig = createTrainConfig("REL", "*", "N", -1337L); + var trainConfig = createTrainConfig("REL", "N", "N", -1337L); var relationshipSplitter = new LinkPredictionRelationshipSampler( graphStore, @@ -386,6 +421,44 @@ void splitWithSpecifiedNegativeRelationships() { //8 * 0.5 = 4 positive, 1 negative assertThat(trainGraphSize).isEqualTo(5); assertThat(featureInputGraphSize).isEqualTo(8); - + Graph outGraph = graphStore.getGraph(trainConfig.nodeLabelIdentifiers(graphStore), List.of(splitConfig.testRelationshipType(), splitConfig.trainRelationshipType()), Optional.of("label")); + var positiveEdgesList = new ArrayList(); + var negativeEdgesList = new ArrayList(); + var idsToNames = IntStream + .range('a', 'o' + 1) + .mapToObj(i -> (char) i) + .collect(Collectors.toMap(c -> idFunction.of(String.valueOf(c)), String::valueOf)); + outGraph.forEachNode(nodeId -> { + outGraph.forEachRelationship(nodeId, -2, (s,t, w) -> { + var relationshipString = "(" + idsToNames.get(outGraph.toOriginalNodeId(s)) + "," + idsToNames.get(outGraph.toOriginalNodeId(t)) + ")"; + if (w == 1.0) { + positiveEdgesList.add(relationshipString); + } + if (w == 0.0) { + negativeEdgesList.add(relationshipString); + } + return true; + }); + return true; + } + ); + assertThat(String.join("\n", positiveEdgesList)) + .isEqualTo( + "(a,b)\n" + + "(a,c)\n" + + "(c,d)\n" + + "(e,g)\n" + + "(f,g)\n" + + "(h,i)\n" + + "(j,k)\n" + + "(j,l)\n" + + "(k,l)\n" + + "(m,n)\n" + + "(m,o)\n" + + "(n,o)"); + assertThat(String.join("\n", negativeEdgesList)) + .isEqualTo("(a,k)\n" + + "(b,k)\n" + + "(c,k)"); } } From e1ad623263835d99ee11ed447240e7ac71e03cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 26 Apr 2023 14:10:00 +0200 Subject: [PATCH 344/400] Cleanup test Avoid string comparisons which are fixed for a specific seed --- ...LinkPredictionRelationshipSamplerTest.java | 71 +++---------------- 1 file changed, 11 insertions(+), 60 deletions(-) diff --git a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java index 0dd392bc1c8..51ed227b268 100644 --- a/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java +++ b/pipeline/src/test/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkPredictionRelationshipSamplerTest.java @@ -24,7 +24,6 @@ import org.neo4j.gds.InspectableTestProgressTracker; import org.neo4j.gds.Orientation; import org.neo4j.gds.RelationshipType; -import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.GraphStore; import org.neo4j.gds.api.schema.ElementSchemaEntry; import org.neo4j.gds.assertj.Extractors; @@ -41,13 +40,11 @@ import org.neo4j.gds.extension.Inject; import org.neo4j.gds.ml.pipeline.linkPipeline.LinkPredictionSplitConfigImpl; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.IntStream; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -62,40 +59,15 @@ class LinkPredictionRelationshipSamplerTest { @GdlGraph(orientation = Orientation.UNDIRECTED, idOffset = 1337) private static final String GRAPH = "CREATE " + - "(x1:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(x2:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(x3:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(x4:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(x5:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(x6:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(x7:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(x8:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(x9:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y1:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y2:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y3:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y4:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y5:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y6:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y7:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y8:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(y9:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(z1:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(z3:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(z5:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(z6:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(z7:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(z8:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "(z9:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + "(a:N {scalar: 0, array: [-1.0, -2.0, 1.0, 1.0, 3.0]}), " + - "(z4:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + "(b:N {scalar: 4, array: [2.0, 1.0, -2.0, 2.0, 1.0]}), " + "(c:N {scalar: 0, array: [-3.0, 4.0, 3.0, 3.0, 2.0]}), " + "(d:N {scalar: 3, array: [1.0, 3.0, 1.0, -1.0, -1.0]}), " + "(e:N {scalar: 1, array: [-2.0, 1.0, 2.0, 1.0, -1.0]}), " + "(f:N {scalar: 0, array: [-1.0, -3.0, 1.0, 2.0, 2.0]}), " + "(g:N {scalar: 1, array: [3.0, 1.0, -3.0, 3.0, 1.0]}), " + - "(z2:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + + // leaving some id gap between nodes + "(:Ignore {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), ".repeat(20) + "(h:N {scalar: 3, array: [-1.0, 3.0, 2.0, 1.0, -3.0]}), " + "(i:N {scalar: 3, array: [4.0, 1.0, 1.0, 2.0, 1.0]}), " + "(j:N {scalar: 4, array: [1.0, -4.0, 2.0, -2.0, 2.0]}), " + @@ -104,7 +76,7 @@ class LinkPredictionRelationshipSamplerTest { "(m:N {scalar: 0, array: [4.0, 4.0, 1.0, 1.0, 1.0]}), " + "(n:N {scalar: 3, array: [1.0, -2.0, 3.0, 2.0, 3.0]}), " + "(o:N {scalar: 2, array: [-3.0, 3.0, -1.0, -1.0, 1.0]}), " + - "" + + "(a)-[:REL {weight: 2.0}]->(b), " + "(a)-[:REL {weight: 1.0}]->(c), " + "(b)-[:REL {weight: 3.0}]->(c), " + @@ -421,44 +393,23 @@ void splitWithSpecifiedNegativeRelationships() { //8 * 0.5 = 4 positive, 1 negative assertThat(trainGraphSize).isEqualTo(5); assertThat(featureInputGraphSize).isEqualTo(8); - Graph outGraph = graphStore.getGraph(trainConfig.nodeLabelIdentifiers(graphStore), List.of(splitConfig.testRelationshipType(), splitConfig.trainRelationshipType()), Optional.of("label")); - var positiveEdgesList = new ArrayList(); - var negativeEdgesList = new ArrayList(); - var idsToNames = IntStream - .range('a', 'o' + 1) - .mapToObj(i -> (char) i) - .collect(Collectors.toMap(c -> idFunction.of(String.valueOf(c)), String::valueOf)); + var outGraph = graphStore.getGraph(trainConfig.nodeLabelIdentifiers(graphStore), List.of(splitConfig.testRelationshipType(), splitConfig.trainRelationshipType()), Optional.of("label")); + + var negativeRelSpace = graphStore.getGraph(RelationshipType.of("NEGATIVE")); + var positiveRelSpace = graphStore.getGraph(RelationshipType.of("REL")); + outGraph.forEachNode(nodeId -> { - outGraph.forEachRelationship(nodeId, -2, (s,t, w) -> { - var relationshipString = "(" + idsToNames.get(outGraph.toOriginalNodeId(s)) + "," + idsToNames.get(outGraph.toOriginalNodeId(t)) + ")"; + outGraph.forEachRelationship(nodeId, Double.NaN, (s,t, w) -> { if (w == 1.0) { - positiveEdgesList.add(relationshipString); + assertThat(positiveRelSpace.exists(outGraph.toRootNodeId(s), outGraph.toRootNodeId(t))).isTrue(); } if (w == 0.0) { - negativeEdgesList.add(relationshipString); + assertThat(negativeRelSpace.exists(outGraph.toRootNodeId(s), outGraph.toRootNodeId(t))).isTrue(); } return true; }); return true; } ); - assertThat(String.join("\n", positiveEdgesList)) - .isEqualTo( - "(a,b)\n" + - "(a,c)\n" + - "(c,d)\n" + - "(e,g)\n" + - "(f,g)\n" + - "(h,i)\n" + - "(j,k)\n" + - "(j,l)\n" + - "(k,l)\n" + - "(m,n)\n" + - "(m,o)\n" + - "(n,o)"); - assertThat(String.join("\n", negativeEdgesList)) - .isEqualTo("(a,k)\n" + - "(b,k)\n" + - "(c,k)"); } } From 1f17224351463a64697e93cc0e094579f6c3fe95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 26 Apr 2023 14:20:11 +0200 Subject: [PATCH 345/400] Fix cherry pick to 2.3 --- .../org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java index 2a4340c5f56..79ec73b7124 100644 --- a/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java +++ b/ml/ml-algo/src/test/java/org/neo4j/gds/ml/splitting/DirectedEdgeSplitterTest.java @@ -286,7 +286,7 @@ void splitWithFilteringWithDifferentSourceTargetLabels() { var selectedRelationships = result.selectedRels().build(); assertThat(selectedRelationships.topology()).satisfies(topology -> { - assertRelSamplingProperties(selectedRelationships, multiLabelGraphStore); + assertRelSamplingProperties(selectedRelationships, multiLabelGraph); assertThat(topology.elementCount()).isEqualTo(1); assertFalse(topology.isMultiGraph()); }); From 2054b56537611a6c91ca797fab66eec08a86e710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 26 Apr 2023 17:02:54 +0200 Subject: [PATCH 346/400] Bump gdsAuraVersion --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 537f719aa24..7d2f4cf77ca 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.4' - gdsAuraVersion = '17' + gdsAuraVersion = '18' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 7e1ac10eaa059b6a7f8c661a81266b5ecd9af4f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 26 Apr 2023 17:05:54 +0200 Subject: [PATCH 347/400] Bump GDS 2.3 version --- README.adoc | 16 ++++++++-------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index 8164888cc3f..1d3d6122d88 100644 --- a/README.adoc +++ b/README.adoc @@ -135,7 +135,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.3.3 + 2.3.4 ---- @@ -147,21 +147,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.3 + 2.3.4 org.neo4j.gds algo - 2.3.3 + 2.3.4 org.neo4j.gds alpha-algo - 2.3.3 + 2.3.4 ---- @@ -173,28 +173,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.3 + 2.3.4 org.neo4j.gds proc - 2.3.3 + 2.3.4 org.neo4j.gds alpha-proc - 2.3.3 + 2.3.4 org.neo4j.gds write-services - 2.3.3 + 2.3.4 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index adc1fdaf899..4449270b8cc 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.4" +| "2.3.5" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index 18061f66d61..afd70dd720e 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.4' + gdsVersion = '2.3.5' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index 7d2f4cf77ca..7946e95501d 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.3.4' + gdsBaseVersion = '2.3.5' gdsAuraVersion = '18' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From 6b5b5040d6fcae3738522547ddfe4678c697c6ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 27 Apr 2023 16:38:22 +0200 Subject: [PATCH 348/400] Bump version --- README.adoc | 16 ++++++++-------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index 1d3d6122d88..d3a9f2a91ae 100644 --- a/README.adoc +++ b/README.adoc @@ -135,7 +135,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.3.4 + 2.3.5 ---- @@ -147,21 +147,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.4 + 2.3.5 org.neo4j.gds algo - 2.3.4 + 2.3.5 org.neo4j.gds alpha-algo - 2.3.4 + 2.3.5 ---- @@ -173,28 +173,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.4 + 2.3.5 org.neo4j.gds proc - 2.3.4 + 2.3.5 org.neo4j.gds alpha-proc - 2.3.4 + 2.3.5 org.neo4j.gds write-services - 2.3.4 + 2.3.5 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 4449270b8cc..6727ad02fdd 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.5" +| "2.3.6" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index afd70dd720e..a213e2d1f54 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.5' + gdsVersion = '2.3.6' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index 7946e95501d..988ca9ee4b0 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.3.5' + gdsBaseVersion = '2.3.6' gdsAuraVersion = '18' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From 5c45b55670aa2e402bc9448d38c7cc2afa7c7b6b Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Wed, 3 May 2023 10:41:18 +0100 Subject: [PATCH 349/400] Rewrite Arrow import process section --- .../pages/graph-project-apache-arrow.adoc | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc index 8975dddd948..2b041b2ff27 100644 --- a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc +++ b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc @@ -2,7 +2,7 @@ [[graph-project-apache-arrow]] = Projecting graphs using Apache Arrow :description: This chapter explains how to import data using Apache Arrow™ into the Graph Data Science library. - +:star: * include::partial$/operations-reference/alpha-note.adoc[] @@ -55,26 +55,25 @@ See xref:graph-project-apache-arrow.adoc#arrow-send-relationships[Sending relati [[arrow-initialize-import-process]] -== Initializing the Import Process +== Initializing the import process An import process is initialized by sending a Flight action using the action type `CREATE_GRAPH`. The action body is a JSON document containing metadata for the import process: ---- { - name: "my_graph", - database_name: "neo4j", - concurrency: 4, - undirected_relationship_types: [] - inverse_indexed_relationship_types: [] + name: "my_graph", <1> + database_name: "neo4j", <2> + concurrency: 4, <3> + undirected_relationship_types: [] <4> + inverse_indexed_relationship_types: [] <5> } ---- - -The `name` is used to identify the import process, it is also the name of the resulting in-memory graph in the graph catalog. -The `database_name` is used to tell the server on which database the projected graph will be available. -The `concurrency` key is optional, it is used during finalizing the in-memory graph on the server after all data has been received. -The `undirected_relationship_types` key is optional, it is used to declare a number of relationship types as undirected. Relationships with the specified types will be imported as undirected. `*` can be used to declare all relationship types as undirected. -The `inverse_indexed_relationship_types` key is optional, it is used to declare a number of relationship types which will also be indexed in inverse direction. `*` can be used to declare all relationship types as inverse indexed. +<1> Used to identify the import process. It is also the name of the resulting in-memory graph in the graph catalog. +<2> The name of the database on which the projected graph will be available. +<3> (optional) The level of concurrency that will be set on the in-memory graph after all data has been received. +<4> (optional) A list of relationship types that must be imported as undirected. A wildcard (`*`) can be used to include all the types. +<5> (optional) A list of relationship types that must be indexed in inverse direction. A wildcard (`*`) can be used to include all the types. [NOTE] Relationships declared as undirected should only be provided once, i.e. in a single direction. From 25e92a2eb71db01deef531598fc1b54a868a4c80 Mon Sep 17 00:00:00 2001 From: Nicola Vitucci Date: Wed, 3 May 2023 10:42:52 +0100 Subject: [PATCH 350/400] Remove page attribute --- doc/modules/ROOT/pages/graph-project-apache-arrow.adoc | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc index 2b041b2ff27..fa8fa6d515b 100644 --- a/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc +++ b/doc/modules/ROOT/pages/graph-project-apache-arrow.adoc @@ -2,7 +2,6 @@ [[graph-project-apache-arrow]] = Projecting graphs using Apache Arrow :description: This chapter explains how to import data using Apache Arrow™ into the Graph Data Science library. -:star: * include::partial$/operations-reference/alpha-note.adoc[] From 7ec03fe69cdd2385cc827452cd0f9a30cbda1445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 28 Apr 2023 12:25:41 +0200 Subject: [PATCH 351/400] Report metric not per db but globally --- .../gds/core/loading/GraphStoreCatalog.java | 21 ++++++++++++------- .../org/neo4j/gds/utils/ExceptionUtil.java | 9 ++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java index 00c11f4493d..e9c36073b79 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/GraphStoreCatalog.java @@ -24,7 +24,9 @@ import org.neo4j.gds.annotation.ValueClass; import org.neo4j.gds.api.DatabaseId; import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.utils.ExceptionUtil; import org.neo4j.gds.utils.StringJoining; import org.neo4j.logging.Log; @@ -184,18 +186,21 @@ private static void set(GraphProjectConfig config, GraphStore graphStore, boolea return userCatalog; }); - listeners.forEach(listener -> { - try { - listener.onProject(config.username(), graphStore.databaseId().databaseName(), config.graphName()); - } catch (Exception e) { - log.ifPresent(l -> l.warn(String.format( + listeners.forEach(listener -> ExceptionUtil.safeRunWithLogException( + log.orElseGet(Neo4jProxy::testLog), + () -> String.format( Locale.US, "Could not call listener %s on setting the graph %s", listener, config.graphName() - ), e)); - } - }); + ), + () -> listener.onProject( + config.username(), + graphStore.databaseId().databaseName(), + config.graphName() + ) + ) + ); } public static boolean exists(String username, String databaseName, String graphName) { diff --git a/core/src/main/java/org/neo4j/gds/utils/ExceptionUtil.java b/core/src/main/java/org/neo4j/gds/utils/ExceptionUtil.java index aa20a459858..151c7414236 100644 --- a/core/src/main/java/org/neo4j/gds/utils/ExceptionUtil.java +++ b/core/src/main/java/org/neo4j/gds/utils/ExceptionUtil.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.Contract; import org.jetbrains.annotations.Nullable; +import org.neo4j.logging.Log; import java.io.IOException; import java.io.UncheckedIOException; @@ -185,6 +186,14 @@ public static void run(CheckedRunnable runnable) { runnable.run(); } + public static void safeRunWithLogException(Log log, Supplier message, Runnable runnable) { + try { + runnable.run(); + } catch (Exception e) { + log.warn(message.get(), e); + } + } + public static Consumer consumer(CheckedConsumer consumer) { return consumer; } From 70284e3bbcab7cf37117afa2714269b0c9d56a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 3 May 2023 16:55:19 +0200 Subject: [PATCH 352/400] List modifying operators in the graph catalog ops --- .../ROOT/pages/management-ops/graph-catalog-ops.adoc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index a9c6f08c84d..6946ebf49f2 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -41,6 +41,16 @@ This chapter explains the available graph catalog operations. | xref:graph-exists.adoc[gds.graph.exists] | Checks if a named graph is stored in the catalog. |=== +== Modifying the graph catalog + +.Graph catalog modification operations: +[opts=header,cols="1m,1"] +|=== +| Name | Description +| xref:graph-catalog-node-ops.adoc[gds.alpha.graph.nodeLabel.mutate] | Computes and adds a new node label to the graph. +| xref:graph-catalog-relationship-ops.adoc[gds.beta.graph.relationships.toUndirected] | Convert one relationship type of the graph from directed to undirected. +|=== + .Graph catalog export operations: [opts=header,cols="1m,1"] From 46253d18ba35aa574c66c39f3a38625f386a3517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 4 May 2023 09:08:27 +0200 Subject: [PATCH 353/400] Improve wording Co-authored-by: Nicola Vitucci --- doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc index 6946ebf49f2..ba37df270e8 100644 --- a/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc +++ b/doc/modules/ROOT/pages/management-ops/graph-catalog-ops.adoc @@ -43,12 +43,12 @@ This chapter explains the available graph catalog operations. == Modifying the graph catalog -.Graph catalog modification operations: +.Graph catalog update operations: [opts=header,cols="1m,1"] |=== | Name | Description | xref:graph-catalog-node-ops.adoc[gds.alpha.graph.nodeLabel.mutate] | Computes and adds a new node label to the graph. -| xref:graph-catalog-relationship-ops.adoc[gds.beta.graph.relationships.toUndirected] | Convert one relationship type of the graph from directed to undirected. +| xref:graph-catalog-relationship-ops.adoc[gds.beta.graph.relationships.toUndirected] | Converts relationship of a given type in a graph from directed to undirected. |=== From 0c93940972677f9ae43ddab9375c3d8c9e9a6a20 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 4 May 2023 15:14:28 +0200 Subject: [PATCH 354/400] Bump Neo4j 4.4 to 4.4.20 --- README.adoc | 2 +- gradle/dependencies.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index d3a9f2a91ae..36c8248e8fd 100644 --- a/README.adoc +++ b/README.adoc @@ -84,7 +84,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.3.0 |Neo4j 5.4.0 .8+<.^|GDS 2.3.x -|Neo4j 4.4.9 - 4.4.19 +|Neo4j 4.4.9 - 4.4.20 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 53fe1e9c52f..3a60b7ca53a 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.19'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.20'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), From bb56b6066bc4796568dd3706280bbf1c730e5552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6ren=20Reichardt?= Date: Wed, 3 May 2023 14:34:50 +0200 Subject: [PATCH 355/400] Remove unused class --- .../core/cypher/SingleElementIterator.java | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/SingleElementIterator.java diff --git a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/SingleElementIterator.java b/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/SingleElementIterator.java deleted file mode 100644 index 8e871d05b80..00000000000 --- a/cypher/cypher-core/src/main/java/org/neo4j/gds/core/cypher/SingleElementIterator.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.core.cypher; - -import com.carrotsearch.hppc.AbstractIterator; - -public class SingleElementIterator extends AbstractIterator { - - private final T element; - private boolean firstIteration; - - public SingleElementIterator(T element) { - this.element = element; - this.firstIteration = true; - } - - @Override - protected T fetch() { - if (!firstIteration) { - return done(); - } - firstIteration = false; - return element; - } -} From c931f3817aa797774a4221bb3a1e4a5acffee981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 3 May 2023 17:18:48 +0200 Subject: [PATCH 356/400] Use EnterpriseDbCreator instead of raw builder Avoid some flaky tests --- etc/forbidden-apis | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/forbidden-apis b/etc/forbidden-apis index 87ef4e5d7c6..c42cbdbfd18 100644 --- a/etc/forbidden-apis +++ b/etc/forbidden-apis @@ -18,3 +18,5 @@ java.util.ServiceLoader#load(java.lang.Class) @ Must pass explicit class loader org.neo4j.kernel.api.procedure.CallableProcedure @ Implement org.neo4j.gds.compat.CompatCallableProcedure instead and call Neo4jProxy.callableProcedure(compat). org.neo4j.kernel.api.procedure.CallableUserAggregationFunction @ Implement org.neo4j.gds.compat.CompatUserAggregationFunction instead and call Neo4jProxy.callableUserAggregationFunction(compat). + +com.neo4j.dbms.api.EnterpriseDatabaseManagementServiceBuilder @ Use EnterpriseDbCreator instead. From 5bfbdcabc6fa8dc4154a66f51d592c9ce0fcdd83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 4 May 2023 17:17:00 +0200 Subject: [PATCH 357/400] Add global private forbidden-apis list Co-authored-by: Veselin Nikolov --- etc/forbidden-apis | 2 -- gradle/forbidden-apis.gradle | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/forbidden-apis b/etc/forbidden-apis index c42cbdbfd18..87ef4e5d7c6 100644 --- a/etc/forbidden-apis +++ b/etc/forbidden-apis @@ -18,5 +18,3 @@ java.util.ServiceLoader#load(java.lang.Class) @ Must pass explicit class loader org.neo4j.kernel.api.procedure.CallableProcedure @ Implement org.neo4j.gds.compat.CompatCallableProcedure instead and call Neo4jProxy.callableProcedure(compat). org.neo4j.kernel.api.procedure.CallableUserAggregationFunction @ Implement org.neo4j.gds.compat.CompatUserAggregationFunction instead and call Neo4jProxy.callableUserAggregationFunction(compat). - -com.neo4j.dbms.api.EnterpriseDatabaseManagementServiceBuilder @ Use EnterpriseDbCreator instead. diff --git a/gradle/forbidden-apis.gradle b/gradle/forbidden-apis.gradle index 0020e170a22..b75dce44b81 100644 --- a/gradle/forbidden-apis.gradle +++ b/gradle/forbidden-apis.gradle @@ -8,6 +8,7 @@ if (shouldForbiddenApis) { proj.apply plugin: 'de.thetaphi.forbiddenapis' proj.forbiddenApis { signaturesFiles += files("$publicDir/etc/forbidden-apis") + ignoreSignaturesOfMissingClasses = true suppressAnnotations = ["org.neo4j.gds.annotation.SuppressForbidden"] bundledSignatures = ['jdk-system-out'] From 930e749e105bc9fe2d9f4a1464dcb7bb2a6e4e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Tue, 9 May 2023 14:35:37 +0200 Subject: [PATCH 358/400] Publish 5.9 compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Florentin Dörre Co-Authored-By: Yuval Rotenberg --- .../5.8/neo4j-kernel-adapter/build.gradle | 63 ++ .../gds/compat/_58/Neo4jProxyFactoryImpl.java | 44 + .../compat/_58/SettingProxyFactoryImpl.java | 44 + .../compat/_58/BoltTransactionRunnerImpl.java | 97 ++ .../gds/compat/_58/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_58/CompatAccessModeImpl.java | 44 + .../_58/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_58/CompatIndexQueryImpl.java | 31 + .../_58/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_58/CompositeNodeCursorImpl.java | 32 + .../gds/compat/_58/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../gds/compat/_58/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_58/Neo4jProxyImpl.java | 923 ++++++++++++++++++ .../compat/_58/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_58/PartitionedStoreScan.java | 58 ++ .../_58/ReferencePropertyReference.java | 49 + .../compat/_58/ScanBasedStoreScanImpl.java | 40 + .../compat/_58/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_58/SettingProxyImpl.java | 87 ++ .../org/neo4j/gds/compat/_58/TestLogImpl.java | 146 +++ .../compat/_58/VirtualRelationshipImpl.java | 41 + .../5.8/storage-engine-adapter/build.gradle | 66 ++ .../_58/InMemoryStorageEngineFactory.java | 268 +++++ .../_58/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_58/InMemoryCountsStoreImpl.java | 112 +++ .../_58/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_58/InMemoryNodeCursor.java | 82 ++ .../_58/InMemoryNodePropertyCursor.java | 45 + .../compat/_58/InMemoryPropertyCursor.java | 71 ++ .../_58/InMemoryPropertySelectionImpl.java | 55 ++ .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_58/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_58/InMemoryStorageEngineFactory.java | 558 +++++++++++ .../compat/_58/InMemoryStorageEngineImpl.java | 321 ++++++ .../compat/_58/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_58/InMemoryStoreVersion.java | 68 ++ .../_58/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_58/InMemoryVersionCheck.java | 61 ++ .../_58/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_58/StorageEngineProxyImpl.java | 156 +++ .../InMemoryLogVersionRepository58.java | 71 ++ ...InMemoryStorageCommandReaderFactory58.java | 43 + .../InMemoryStorageReader58.java | 332 +++++++ gradle/dependencies.gradle | 1 + .../org/neo4j/gds/compat/Neo4jVersion.java | 7 +- .../neo4j/gds/compat/Neo4jVersionTest.java | 4 +- .../java/org/neo4j/gds/SysInfoProcTest.java | 13 + 51 files changed, 5297 insertions(+), 2 deletions(-) create mode 100644 compatibility/5.8/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableProcedureImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatAccessModeImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatIndexQueryImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/PartitionedStoreScan.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ReferencePropertyReference.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/TestLogImpl.java create mode 100644 compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/VirtualRelationshipImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodeCursor.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertyCursor.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStoreVersion.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryVersionCheck.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository58.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory58.java create mode 100644 compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader58.java diff --git a/compatibility/5.8/neo4j-kernel-adapter/build.gradle b/compatibility/5.8/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..44bde179dab --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.8' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.8' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.8' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.8' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.8' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.8' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.8' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..2ddab471d98 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.8 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.8 (placeholder)"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..cf7d81502e2 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.8 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.8 (placeholder)"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/BoltTransactionRunnerImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..1e8ee114528 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableProcedureImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableProcedureImpl.java new file mode 100644 index 00000000000..3dfbea82de7 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableUserAggregationFunctionImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..bdcafd01e01 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatAccessModeImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatAccessModeImpl.java new file mode 100644 index 00000000000..97ed5b9e6f1 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatGraphDatabaseAPIImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..12d98ba7f9b --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatIndexQueryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..bdc9c206219 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatUsernameAuthSubjectImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..a3b2bdb802a --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompositeNodeCursorImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..8e6ccc18856 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseLayoutImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..7108a397ba9 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..edafd3e92dd --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..47ec85af7af --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_8; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.8"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java new file mode 100644 index 00000000000..5dff13d4a28 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java @@ -0,0 +1,923 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/NodeLabelIndexLookupImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..13ea95378a2 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/PartitionedStoreScan.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/PartitionedStoreScan.java new file mode 100644 index 00000000000..b4a77f3bf84 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ReferencePropertyReference.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ReferencePropertyReference.java new file mode 100644 index 00000000000..9296154116d --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ScanBasedStoreScanImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..f3ae18f5fec --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..847538956a6 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_8; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.8"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyImpl.java new file mode 100644 index 00000000000..a6a5cafa274 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/TestLogImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/TestLogImpl.java new file mode 100644 index 00000000000..ba92ebb85eb --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/VirtualRelationshipImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..25287a363f6 --- /dev/null +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/build.gradle b/compatibility/5.8/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..e2d02c8f29b --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.8' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.8' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.8' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.8' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.8' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.8' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java b/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..19dbb9dfeac --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported58"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..ea12427918f --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.8 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.8"; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCommandCreationContextImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..abf59f71994 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCountsStoreImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..bc256458512 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryMetaDataProviderImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..c7eb156c0c6 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository58; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository58 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository58(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodeCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodeCursor.java new file mode 100644 index 00000000000..55857a73fc4 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodePropertyCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..079cda66b29 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertyCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..91f7dd25f1c --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertySelectionImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..43ce0c8d484 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipPropertyCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..4c3a5eb810b --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipScanCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..71cd9fcdc19 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipTraversalCursor.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..5c82c1e02ac --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..36a4ff16c76 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineFactory.java @@ -0,0 +1,558 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory58; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.Locks; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-58"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_8, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory58.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public Locks createLocks(Config config, SystemNanoClock clock) { + return Locks.NO_LOCKS; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..f23b9764315 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageEngineImpl.java @@ -0,0 +1,321 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader58; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + throw new UnsupportedOperationException(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader58(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageLocksImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..8ef6bfc8b53 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStoreVersion.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStoreVersion.java new file mode 100644 index 00000000000..37e1f46f25d --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryTransactionIdStoreImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..b607e28637e --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryVersionCheck.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryVersionCheck.java new file mode 100644 index 00000000000..187731326b5 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._58.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..c8f0a8393b8 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_8; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.8"; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..c47d09fd394 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_58/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._58; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader58; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader58(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository58.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository58.java new file mode 100644 index 00000000000..e495c490b52 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository58.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository58 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository58() { + this(0, 0); + } + + private InMemoryLogVersionRepository58(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory58.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory58.java new file mode 100644 index 00000000000..5b457388c12 --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory58.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory58 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory58(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader58.java b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader58.java new file mode 100644 index 00000000000..3db83ec93cf --- /dev/null +++ b/compatibility/5.8/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader58.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._58.InMemoryNodeCursor; +import org.neo4j.gds.compat._58.InMemoryPropertyCursor; +import org.neo4j.gds.compat._58.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._58.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader58 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader58( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 3a60b7ca53a..56f53abbd90 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -8,6 +8,7 @@ ext { '5.5': properties.getOrDefault('neo4jVersion55', '5.5.0'), '5.6': properties.getOrDefault('neo4jVersion56', '5.6.0'), '5.7': properties.getOrDefault('neo4jVersion57', '5.7.0'), + '5.8': properties.getOrDefault('neo4jVersion58', '5.8.0'), ] neo4jDefault = neos.'4.4' diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 2593a05bd1d..6f31ca9ab5c 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -35,6 +35,7 @@ public enum Neo4jVersion { V_5_5, V_5_6, V_5_7, + V_5_8, V_RC; @Override @@ -56,6 +57,8 @@ public String toString() { return "5.6"; case V_5_7: return "5.7"; + case V_5_8: + return "5.8"; case V_RC: return "rc"; default: @@ -65,7 +68,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 8); + return ImmutableMajorMinorVersion.of(5, 9); } String version = toString(); var subVersions = version.split("\\."); @@ -142,6 +145,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 7) { return Neo4jVersion.V_5_7; } else if (minorVersion == 8) { + return Neo4jVersion.V_5_8; + } else if (minorVersion == 9) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 3b940016f6a..999179b6812 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -48,7 +48,8 @@ class Neo4jVersionTest { "5.5.0, V_5_5", "5.6.0, V_5_6", "5.7.0, V_5_7", - "5.8.0, V_RC", + "5.8.0, V_5_8", + "5.9.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); @@ -88,6 +89,7 @@ void shouldNotRespectVersionOverride() { "5.5.0, 5, 5", "5.6.0, 5, 6", "5.7.0, 5, 7", + "5.8.0, 5, 8", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 69895a17779..0c5e0e4d2be 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -86,6 +86,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.7", "Neo4j Settings 5.7 (placeholder)", + "Neo4j 5.8", + "Neo4j 5.8 (placeholder)", + "Neo4j Settings 5.8", + "Neo4j Settings 5.8 (placeholder)", + "Neo4j RC", "Neo4j RC (placeholder)", "Neo4j Settings RC", @@ -197,6 +202,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.7" ); break; + case V_5_8: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.8 (placeholder)", + "Neo4j Settings 5.8", + "Neo4j 5.8 (placeholder)", + "Neo4j 5.8" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC (placeholder)", From d2355d21cf2fd20092935b1a2c04ed8271c23ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 24 Apr 2023 11:37:08 +0200 Subject: [PATCH 359/400] Move isCompositeDB behind Neo4j Proxy 4.4 takes a String but 5.8.0 (dev) only takes a NamedDatabaseId --- .../neo4j/gds/compat/_44/Neo4jProxyImpl.java | 8 +++++ .../neo4j/gds/compat/_51/Neo4jProxyImpl.java | 7 ++++ .../neo4j/gds/compat/_52/Neo4jProxyImpl.java | 7 ++++ .../neo4j/gds/compat/_53/Neo4jProxyImpl.java | 7 ++++ .../neo4j/gds/compat/_54/Neo4jProxyImpl.java | 7 ++++ .../neo4j/gds/compat/_55/Neo4jProxyImpl.java | 7 ++++ .../neo4j/gds/compat/_56/Neo4jProxyImpl.java | 7 ++++ .../neo4j/gds/compat/_57/Neo4jProxyImpl.java | 7 ++++ .../org/neo4j/gds/compat/Neo4jProxyApi.java | 2 ++ .../java/org/neo4j/gds/compat/Neo4jProxy.java | 4 +++ .../gds/projection/CypherAggregation.java | 2 +- .../projection/DatabaseTopologyHelper.java | 34 ------------------- 12 files changed, 64 insertions(+), 35 deletions(-) delete mode 100644 cypher-aggregation/src/main/java/org/neo4j/gds/projection/DatabaseTopologyHelper.java diff --git a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java index c6b32cbccd1..64bf9b8c087 100644 --- a/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java +++ b/compatibility/4.4/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_44/Neo4jProxyImpl.java @@ -28,6 +28,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -40,6 +41,7 @@ import org.neo4j.gds.compat.GdsDatabaseLayout; import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; import org.neo4j.gds.compat.InputEntityIdVisitor; import org.neo4j.gds.compat.Neo4jProxyApi; import org.neo4j.gds.compat.PropertyReference; @@ -806,4 +808,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService).name()); + } } diff --git a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java index 83a1ac02da9..17bfd3e1bf0 100644 --- a/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java +++ b/compatibility/5.1/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_51/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -925,4 +926,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java index d8ed7eae7a7..18178a08351 100644 --- a/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java +++ b/compatibility/5.2/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_52/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -923,4 +924,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java index 4902182be1e..22542193d48 100644 --- a/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java +++ b/compatibility/5.3/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_53/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -924,4 +925,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java index 32662efb8fc..95a20fe74d7 100644 --- a/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java +++ b/compatibility/5.4/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_54/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -924,4 +925,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java index be8b09f7925..099e4b3e576 100644 --- a/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java +++ b/compatibility/5.5/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_55/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -925,4 +926,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java index bd4ba3f4682..1458cdca72a 100644 --- a/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java +++ b/compatibility/5.6/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_56/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -925,4 +926,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java index dbc5e0e2c59..aee4d24f8ce 100644 --- a/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java +++ b/compatibility/5.7/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_57/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -920,4 +921,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } diff --git a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java index 4cddd86ccea..ab22f31b4d4 100644 --- a/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java +++ b/compatibility/api/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxyApi.java @@ -322,4 +322,6 @@ TransactionalContext newQueryContext( String queryText, MapValue queryParameters ); + + boolean isCompositeDatabase(GraphDatabaseService databaseService); } diff --git a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java index f081d37c75b..8a2490bb865 100644 --- a/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java +++ b/compatibility/common/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/Neo4jProxy.java @@ -484,6 +484,10 @@ public static TransactionalContext newQueryContext( return IMPL.newQueryContext(contextFactory, tx, queryText, queryParameters); } + public static boolean isCompositeDatabase(GraphDatabaseService databaseService) { + return IMPL.isCompositeDatabase(databaseService); + } + private Neo4jProxy() { throw new UnsupportedOperationException("No instances"); } diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java index 1125bc0ade1..c348737a88e 100644 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java +++ b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/CypherAggregation.java @@ -92,7 +92,7 @@ public CompatUserAggregator create(Context ctx) throws ProcedureException { .apply(ctx); var username = procedures.lookupComponentProvider(Username.class, true).apply(ctx); - var runsOnCompositeDatabase = DatabaseTopologyHelper.isCompositeDatabase(databaseService); + var runsOnCompositeDatabase = Neo4jProxy.isCompositeDatabase(databaseService); return new GraphAggregator( DatabaseId.of(databaseService), diff --git a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/DatabaseTopologyHelper.java b/cypher-aggregation/src/main/java/org/neo4j/gds/projection/DatabaseTopologyHelper.java deleted file mode 100644 index 9583ad90f09..00000000000 --- a/cypher-aggregation/src/main/java/org/neo4j/gds/projection/DatabaseTopologyHelper.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [http://neo4j.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.gds.projection; - -import org.neo4j.fabric.FabricDatabaseManager; -import org.neo4j.gds.compat.GraphDatabaseApiProxy; -import org.neo4j.graphdb.GraphDatabaseService; - -public final class DatabaseTopologyHelper { - - private DatabaseTopologyHelper() {} - - public static boolean isCompositeDatabase(GraphDatabaseService graphDatabaseService) { - var databaseManager = GraphDatabaseApiProxy.resolveDependency(graphDatabaseService, FabricDatabaseManager.class); - return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(graphDatabaseService).name()); - } -} From 71d9bc3cd4d5e42f6001463ab372e0dc2e643b1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 9 May 2023 15:05:50 +0200 Subject: [PATCH 360/400] Fix 5.8 compat layer --- .../java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java index 5dff13d4a28..1a8a9c75af6 100644 --- a/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java +++ b/compatibility/5.8/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_58/Neo4jProxyImpl.java @@ -31,6 +31,7 @@ import org.neo4j.configuration.helpers.DatabaseNameValidator; import org.neo4j.dbms.api.DatabaseManagementService; import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; import org.neo4j.gds.annotation.SuppressForbidden; import org.neo4j.gds.compat.BoltTransactionRunner; import org.neo4j.gds.compat.CompatCallableProcedure; @@ -920,4 +921,10 @@ public TransactionalContext newQueryContext( ) { return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } } From d2c11175cf6a9b69f20e2be876a0a63890477153 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 10 May 2023 17:39:51 +0200 Subject: [PATCH 361/400] correct place --- doc/modules/ROOT/pages/algorithms/centrality.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/centrality.adoc b/doc/modules/ROOT/pages/algorithms/centrality.adoc index 4ecb475265a..0d04f0b6359 100644 --- a/doc/modules/ROOT/pages/algorithms/centrality.adoc +++ b/doc/modules/ROOT/pages/algorithms/centrality.adoc @@ -14,7 +14,8 @@ The Neo4j GDS library includes the following centrality algorithms, grouped by q ** xref:algorithms/degree-centrality.adoc[Degree Centrality] * Beta ** xref:algorithms/closeness-centrality.adoc[Closeness Centrality] +** xref:algorithms/celf.adoc[CELF] * Alpha ** xref:algorithms/harmonic-centrality.adoc[Harmonic Centrality] ** xref:algorithms/hits.adoc[HITS] -** xref:algorithms/celf.adoc[CELF] + From c048a490cb0562d930d937f82f885bf81d841715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 11 May 2023 15:09:20 +0200 Subject: [PATCH 362/400] Bump GDS Aura version the last one was broken --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 988ca9ee4b0..a99f3b119dc 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.6' - gdsAuraVersion = '18' + gdsAuraVersion = '19' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 0a0cff08a0d7721243dd659d2f83f85d6953e8b0 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 11 May 2023 12:26:27 +0200 Subject: [PATCH 363/400] Split EE section to separate file --- doc/modules/ROOT/pages/introduction.adoc | 28 +------------------ .../introduction/enterprise-features.adoc | 26 +++++++++++++++++ 2 files changed, 27 insertions(+), 27 deletions(-) create mode 100644 doc/modules/ROOT/partials/introduction/enterprise-features.adoc diff --git a/doc/modules/ROOT/pages/introduction.adoc b/doc/modules/ROOT/pages/introduction.adoc index 1455bba2af5..ab7e071502c 100644 --- a/doc/modules/ROOT/pages/introduction.adoc +++ b/doc/modules/ROOT/pages/introduction.adoc @@ -83,30 +83,4 @@ The amount of data loaded can be controlled by so called graph projections, whic For more information see xref:management-ops/index.adoc[Graph Management]. - -[[introduction-editions]] -== Editions - -The Neo4j Graph Data Science library is available in two editions. - -* The open source Community Edition: -** Includes all algorithms. -** Most of the catalog operations to manage graphs, models and pipelines are available. Unavailable operations are listed below. -** Limits the concurrency to 4 CPU cores. -** Limits the capacity of the model catalog to 4 models. -* The Neo4j Graph Data Science library Enterprise Edition: -** Can run on an unlimited amount of CPU cores. -** Supports the role-based access control system (RBAC) from Neo4j Enterprise Edition. -** Support running GDS as part of a xref::production-deployment/neo4j-cluster.adoc[Neo4j cluster deployment]. -** Includes capacity and load xref::common-usage/monitoring-system.adoc[monitoring]. -** Supports various additional graph catalog features, including: -*** Graph xref::management-ops/backup-restore.adoc[backup and restore]. -*** Data import and export via xref:installation/installation-apache-arrow.adoc[Apache Arrow]. -** Supports various additional model catalog features, including: -*** Storing unlimited amounts of models in the model catalog. -*** Sharing of models between users, by xref:model-catalog/publish.adoc[publishing it]. -*** Model xref:model-catalog/store.adoc#model-catalog-store-ops[persistence to disk]. -** Supports an xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[optimized graph implementation]. -** Allows the configuration of xref:production-deployment/defaults-and-limits.adoc[defaults and limits]. - -For more information see xref:installation/System-requirements.adoc#system-requirements-cpu[System Requirements - CPU]. +include::partial$/introduction/enterprise-features.adoc[leveloffset=+1] diff --git a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc new file mode 100644 index 00000000000..c60e8b4caf7 --- /dev/null +++ b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc @@ -0,0 +1,26 @@ +[[introduction-editions]] += Editions + +The Neo4j Graph Data Science library is available in two editions. + +* The open source Community Edition: +** Includes all algorithms. +** Most of the catalog operations to manage graphs, models and pipelines are available. Unavailable operations are listed below. +** Limits the concurrency to 4 CPU cores. +** Limits the capacity of the model catalog to 4 models. +* The Neo4j Graph Data Science library Enterprise Edition: +** Can run on an unlimited amount of CPU cores. +** Supports the role-based access control system (RBAC) from Neo4j Enterprise Edition. +** Support running GDS as part of a xref::production-deployment/neo4j-cluster.adoc[Neo4j cluster deployment]. +** Includes capacity and load xref::common-usage/monitoring-system.adoc[monitoring]. +** Supports various additional graph catalog features, including: +*** Graph xref::management-ops/backup-restore.adoc[backup and restore]. +*** Data import and export via xref:installation/installation-apache-arrow.adoc[Apache Arrow]. +** Supports various additional model catalog features, including: +*** Storing unlimited amounts of models in the model catalog. +*** Sharing of models between users, by xref:model-catalog/publish.adoc[publishing it]. +*** Model xref:model-catalog/store.adoc#model-catalog-store-ops[persistence to disk]. +** Supports an xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[optimized graph implementation]. +** Allows the configuration of xref:production-deployment/defaults-and-limits.adoc[defaults and limits]. + +For more information see xref:installation/System-requirements.adoc#system-requirements-cpu[System Requirements - CPU]. From 4e2b237376c3964e08aace2999af55cedd71ba7c Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 11 May 2023 12:06:21 +0200 Subject: [PATCH 364/400] Correctly document EE features - RBAC is free - model catalog is limited at 3 - cluster integration works, except writes And make the language more consistent --- .../introduction/enterprise-features.adoc | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc index c60e8b4caf7..a317aa9468a 100644 --- a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc +++ b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc @@ -5,22 +5,23 @@ The Neo4j Graph Data Science library is available in two editions. * The open source Community Edition: ** Includes all algorithms. -** Most of the catalog operations to manage graphs, models and pipelines are available. Unavailable operations are listed below. +** Limits the catalog operations to manage graphs and models. +Unavailable operations are listed under the Enterprise Edition below. ** Limits the concurrency to 4 CPU cores. -** Limits the capacity of the model catalog to 4 models. +** Limits the capacity of the model catalog to 3 models. + * The Neo4j Graph Data Science library Enterprise Edition: -** Can run on an unlimited amount of CPU cores. -** Supports the role-based access control system (RBAC) from Neo4j Enterprise Edition. -** Support running GDS as part of a xref::production-deployment/neo4j-cluster.adoc[Neo4j cluster deployment]. -** Includes capacity and load xref::common-usage/monitoring-system.adoc[monitoring]. -** Supports various additional graph catalog features, including: +** Supports running on any amount of CPU cores. +** Supports running GDS write workloads as part of a xref::production-deployment/neo4j-cluster.adoc[Neo4j cluster deployment]. +** Supports capacity and load xref::common-usage/monitoring-system.adoc[monitoring]. +** Supports extended graph catalog features, including: *** Graph xref::management-ops/backup-restore.adoc[backup and restore]. *** Data import and export via xref:installation/installation-apache-arrow.adoc[Apache Arrow]. -** Supports various additional model catalog features, including: -*** Storing unlimited amounts of models in the model catalog. -*** Sharing of models between users, by xref:model-catalog/publish.adoc[publishing it]. +** Supports extended model catalog features, including: +*** Storing any number of models in the model catalog. +*** Sharing of models between users, through xref:model-catalog/publish.adoc[publishing]. *** Model xref:model-catalog/store.adoc#model-catalog-store-ops[persistence to disk]. -** Supports an xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[optimized graph implementation]. -** Allows the configuration of xref:production-deployment/defaults-and-limits.adoc[defaults and limits]. +** Supports an xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[optimized graph implementation], enabled by default. +** Supports the configuration of xref:production-deployment/defaults-and-limits.adoc[defaults and limits]. For more information see xref:installation/System-requirements.adoc#system-requirements-cpu[System Requirements - CPU]. From 53750dac8a848f764a8f5ef273a31138302adc45 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 11 May 2023 12:06:37 +0200 Subject: [PATCH 365/400] Reference the default and how to enable EE --- .../ROOT/partials/introduction/enterprise-features.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc index a317aa9468a..8bee5f03461 100644 --- a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc +++ b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc @@ -2,6 +2,9 @@ = Editions The Neo4j Graph Data Science library is available in two editions. +By default, GDS will operate as the Community Edition. +To unlock Enterprise Edition features, a valid Neo4j Graph Data Science Enterprise license file is required. +See xref:operations-reference/configuration-settings.adoc#gds.enterprise.license_file[Configuration Settings] for how to configure the license. * The open source Community Edition: ** Includes all algorithms. From 2a506d4c25c6095088360ebdd40c415aa3dd3ca2 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 11 May 2023 12:09:02 +0200 Subject: [PATCH 366/400] Inline link to CPU section --- .../ROOT/partials/introduction/enterprise-features.adoc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc index 8bee5f03461..0191ded3a3b 100644 --- a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc +++ b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc @@ -10,11 +10,11 @@ See xref:operations-reference/configuration-settings.adoc#gds.enterprise.license ** Includes all algorithms. ** Limits the catalog operations to manage graphs and models. Unavailable operations are listed under the Enterprise Edition below. -** Limits the concurrency to 4 CPU cores. +** Limits the xref:installation/System-requirements.adoc#system-requirements-cpu[concurrency to 4 CPU cores]. ** Limits the capacity of the model catalog to 3 models. * The Neo4j Graph Data Science library Enterprise Edition: -** Supports running on any amount of CPU cores. +** Supports running on xref:installation/System-requirements.adoc#system-requirements-cpu[any amount of CPU cores]. ** Supports running GDS write workloads as part of a xref::production-deployment/neo4j-cluster.adoc[Neo4j cluster deployment]. ** Supports capacity and load xref::common-usage/monitoring-system.adoc[monitoring]. ** Supports extended graph catalog features, including: @@ -26,5 +26,3 @@ Unavailable operations are listed under the Enterprise Edition below. *** Model xref:model-catalog/store.adoc#model-catalog-store-ops[persistence to disk]. ** Supports an xref:production-deployment/feature-toggles.adoc#bit-id-map-feature-toggle[optimized graph implementation], enabled by default. ** Supports the configuration of xref:production-deployment/defaults-and-limits.adoc[defaults and limits]. - -For more information see xref:installation/System-requirements.adoc#system-requirements-cpu[System Requirements - CPU]. From f42690deffc8395a625b4da2ccedf2705f34f44e Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 11 May 2023 12:18:27 +0200 Subject: [PATCH 367/400] Update license registration link --- doc/modules/ROOT/pages/installation/System-requirements.adoc | 2 +- .../pages/installation/installation-enterprise-edition.adoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/pages/installation/System-requirements.adoc b/doc/modules/ROOT/pages/installation/System-requirements.adoc index 5b1db524424..9df47aa81a7 100644 --- a/doc/modules/ROOT/pages/installation/System-requirements.adoc +++ b/doc/modules/ROOT/pages/installation/System-requirements.adoc @@ -88,6 +88,6 @@ The maximum concurrency that can be used is limited depending on the license und * Neo4j Graph Data Science Library - Enterprise Edition (GDS EE) ** The maximum concurrency in the library is unlimited. -To register for a license, please contact Neo4j at https://neo4j.com/contact-us/?ref=graph-data-science. +To register for a license, please visit https://neo4j.com/contact-us/?ref=graph-data-science[neo4j.com]. NOTE: Concurrency limits are determined based on whether you have a GDS EE license, or if you are using GDS CE. The maximum concurrency limit in the graph data science library is not set based on your edition of the Neo4j database. diff --git a/doc/modules/ROOT/pages/installation/installation-enterprise-edition.adoc b/doc/modules/ROOT/pages/installation/installation-enterprise-edition.adoc index 232d9245c82..356461f0574 100644 --- a/doc/modules/ROOT/pages/installation/installation-enterprise-edition.adoc +++ b/doc/modules/ROOT/pages/installation/installation-enterprise-edition.adoc @@ -2,7 +2,7 @@ = Enterprise Edition Configuration Unlocking the Enterprise Edition of the Neo4j Graph Data Science library requires a valid license key. -To register for a license, please contact Neo4j at https://neo4j.com/contact-us/?ref=graph-analytics. +To register for a license, please visit https://neo4j.com/contact-us/?ref=graph-data-science[neo4j.com]. The license is issued in the form of a license key file, which needs to be placed in a directory accessible by the Neo4j server. You can configure the location of the license key file by setting the `gds.enterprise.license_file` option in the `neo4j.conf` configuration file of your Neo4j installation. From df06e251158e2420bc86c5410ee62676e65c582b Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 11 May 2023 12:18:41 +0200 Subject: [PATCH 368/400] Link to Installation chapter instead --- .../ROOT/partials/introduction/enterprise-features.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc index 0191ded3a3b..21f3ac72383 100644 --- a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc +++ b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc @@ -4,12 +4,12 @@ The Neo4j Graph Data Science library is available in two editions. By default, GDS will operate as the Community Edition. To unlock Enterprise Edition features, a valid Neo4j Graph Data Science Enterprise license file is required. -See xref:operations-reference/configuration-settings.adoc#gds.enterprise.license_file[Configuration Settings] for how to configure the license. +See xref:installation/installation-enterprise-edition.adoc[] for how to configure the license. * The open source Community Edition: ** Includes all algorithms. ** Limits the catalog operations to manage graphs and models. -Unavailable operations are listed under the Enterprise Edition below. + Unavailable operations are listed under the Enterprise Edition below. ** Limits the xref:installation/System-requirements.adoc#system-requirements-cpu[concurrency to 4 CPU cores]. ** Limits the capacity of the model catalog to 3 models. From e2b9b6558b10b9d2a0d524a6eb4a1e3fadc2b113 Mon Sep 17 00:00:00 2001 From: yuval Date: Fri, 12 May 2023 11:34:35 +0200 Subject: [PATCH 369/400] bump gdsAuraVersion to 20 --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index a99f3b119dc..c6264e70382 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.6' - gdsAuraVersion = '19' + gdsAuraVersion = '20' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 04948d5a0b64f156014857ed812a9a17710f8841 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 12 May 2023 16:07:06 +0200 Subject: [PATCH 370/400] Clarify concurrency limitation Co-authored-by: Paul Horn --- doc/modules/ROOT/partials/introduction/enterprise-features.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc index 21f3ac72383..2e9f427f58e 100644 --- a/doc/modules/ROOT/partials/introduction/enterprise-features.adoc +++ b/doc/modules/ROOT/partials/introduction/enterprise-features.adoc @@ -10,7 +10,7 @@ See xref:installation/installation-enterprise-edition.adoc[] for how to configur ** Includes all algorithms. ** Limits the catalog operations to manage graphs and models. Unavailable operations are listed under the Enterprise Edition below. -** Limits the xref:installation/System-requirements.adoc#system-requirements-cpu[concurrency to 4 CPU cores]. +** Limits the xref:installation/System-requirements.adoc#system-requirements-cpu[concurrency to maximum 4 CPU cores]. ** Limits the capacity of the model catalog to 3 models. * The Neo4j Graph Data Science library Enterprise Edition: From 0a6b9bded56e050bcfae4b510f87f5d2a104e524 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 12 May 2023 16:03:03 +0200 Subject: [PATCH 371/400] Fix estimation issue in HashGNN --- .../embeddings/hashgnn/HashGNNFactory.java | 2 +- .../gds/embeddings/hashgnn/HashGNNTest.java | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNFactory.java b/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNFactory.java index f58ebcca91e..dacde0b0137 100644 --- a/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNFactory.java +++ b/algo/src/main/java/org/neo4j/gds/embeddings/hashgnn/HashGNNFactory.java @@ -118,7 +118,7 @@ public MemoryEstimation memoryEstimation(CONFIG config) { config.heterogeneous() ? dims.relationshipCounts().size() : 1 ))); - int outputDimension = config.outputDimension().orElse(FUDGED_BINARY_DIMENSION); + int outputDimension = config.outputDimension().orElse(binaryDimension); builder.perNode( "Embeddings output", n -> HugeObjectArray.memoryEstimation(n, MemoryUsage.sizeOfDoubleArray(outputDimension)) diff --git a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java index 083c6bd4b17..32d8e88339f 100644 --- a/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java +++ b/algo/src/test/java/org/neo4j/gds/embeddings/hashgnn/HashGNNTest.java @@ -35,6 +35,7 @@ import org.neo4j.gds.collections.HugeSparseLongArray; import org.neo4j.gds.compat.Neo4jProxy; import org.neo4j.gds.compat.TestLog; +import org.neo4j.gds.core.GraphDimensions; import org.neo4j.gds.core.concurrency.Pools; import org.neo4j.gds.core.loading.ArrayIdMap; import org.neo4j.gds.core.loading.LabelInformationBuilders; @@ -312,6 +313,37 @@ void shouldEstimateMemory( ); } + @Test + void estimationShouldUseGeneratedDimensionIfOutputIsMissing() { + var inputDimension = 1000L; + var inputRatio = 0.1; + var graphDims = GraphDimensions.of((long) 1e6); + var concurrency = 4; + + var bigEstimation = new HashGNNFactory<>() + .memoryEstimation(HashGNNStreamConfigImpl + .builder() + .generateFeatures(Map.of("dimension", inputDimension, "densityLevel", 1)) + .iterations(3) + .embeddingDensity(100) + .build()) + .estimate(graphDims, concurrency) + .memoryUsage(); + + var smallEstimation = new HashGNNFactory<>() + .memoryEstimation(HashGNNStreamConfigImpl + .builder() + .generateFeatures(Map.of("dimension", (long) (inputRatio * inputDimension), "densityLevel", 1)) + .iterations(3) + .embeddingDensity(100) + .build()) + .estimate(graphDims, concurrency) + .memoryUsage(); + + var outputRatio = (double) smallEstimation.min / bigEstimation.min; + assertThat(outputRatio).isCloseTo(inputRatio, Offset.offset(0.1)); + } + @ParameterizedTest @CsvSource(value = {"true", "false"}) void shouldLogProgress(boolean dense) { From 63eb9a846fa70ff8ede1e3e6b79ff56c99a79a08 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Fri, 12 May 2023 16:03:17 +0200 Subject: [PATCH 372/400] Add missing space in estimation tree for LinkPrediction --- .../linkPipeline/train/LinkFeaturesAndLabelsExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkFeaturesAndLabelsExtractor.java b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkFeaturesAndLabelsExtractor.java index 289f1a4a1ae..63f087cf9a5 100644 --- a/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkFeaturesAndLabelsExtractor.java +++ b/pipeline/src/main/java/org/neo4j/gds/ml/pipeline/linkPipeline/train/LinkFeaturesAndLabelsExtractor.java @@ -64,7 +64,7 @@ static MemoryEstimation estimate( .times(relSetSizeExtractor.applyAsLong(graphDim.relationshipCounts())) .add(MemoryUsage.sizeOfInstance(HugeObjectArray.class))) .perGraphDimension( - setDesc + "relationship targets", + setDesc + " relationship targets", (graphDim, threads) -> MemoryRange.of( HugeIntArray.memoryEstimation(relSetSizeExtractor.applyAsLong(graphDim.relationshipCounts())) ) From 59f073b11a8a37d77360a8b3743973a4ef6cb4ea Mon Sep 17 00:00:00 2001 From: Brian Shi Date: Mon, 15 May 2023 11:46:04 +0100 Subject: [PATCH 373/400] Fix hashgnn doc memory estimation --- .../ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc index beb25ddca08..08b2e7ec7ec 100644 --- a/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc +++ b/doc/modules/ROOT/pages/machine-learning/node-embeddings/hashgnn.adoc @@ -457,7 +457,7 @@ YIELD nodeCount, relationshipCount, bytesMin, bytesMax, requiredMemory [opts="header", cols="1,1,1,1,1"] |=== | nodeCount | relationshipCount | bytesMin | bytesMax | requiredMemory -| 7 | 18 | 59160 | 59160 | "57 KiB" +| 7 | 18 | 2040 | 2040 | "2040 Bytes" |=== -- From 4a072b0c9113af91f4bfebaaeeff898c56b9188b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Tue, 16 May 2023 15:41:48 +0200 Subject: [PATCH 374/400] Add scalas and log4j entries for future 5.9 --- gradle/dependencies.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 56f53abbd90..6f6e9c6c515 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -26,6 +26,7 @@ ext { '5.6': '2.13.8', '5.7': '2.13.10', '5.8': '2.13.10', + '5.9': '2.13.10', ] log4js = [ @@ -38,6 +39,7 @@ ext { '5.6': '2.19.0', '5.7': '2.20.0', '5.8': '2.20.0', + '5.9': '2.20.0', ] ver = [ From 22d4b3c94430cd3ee7f5a922bbbebde37388f6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 17 May 2023 09:05:44 +0200 Subject: [PATCH 375/400] Update version to 2.3.7 --- README.adoc | 17 +++++++++-------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index 36c8248e8fd..20d11acd34f 100644 --- a/README.adoc +++ b/README.adoc @@ -92,6 +92,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.5.0 |Neo4j 5.6.0 |Neo4j 5.7.0 +|Neo4j 5.8.0 |=== NOTE: Preview releases are not automatically made available in Neo4j Desktop. They need to be installed manually. @@ -135,7 +136,7 @@ For the most basic set of features, like graph loading and the graph representat org.neo4j.gds core - 2.3.5 + 2.3.6 ---- @@ -147,21 +148,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.5 + 2.3.6 org.neo4j.gds algo - 2.3.5 + 2.3.6 org.neo4j.gds alpha-algo - 2.3.5 + 2.3.6 ---- @@ -173,28 +174,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.5 + 2.3.6 org.neo4j.gds proc - 2.3.5 + 2.3.6 org.neo4j.gds alpha-proc - 2.3.5 + 2.3.6 org.neo4j.gds write-services - 2.3.5 + 2.3.6 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 6727ad02fdd..545d7e209bc 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.6" +| "2.3.7" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index a213e2d1f54..970a31c4037 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.6' + gdsVersion = '2.3.7' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index c6264e70382..55a819da2e7 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,5 +1,5 @@ ext { - gdsBaseVersion = '2.3.6' + gdsBaseVersion = '2.3.7' gdsAuraVersion = '20' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") From 621957b6c6fdce827a8fbc53e787b4ace1483d67 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 25 May 2023 10:08:51 +0200 Subject: [PATCH 376/400] Bump to 4.4.21 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 6f6e9c6c515..7e41ca69b83 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.20'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.21'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), From 69464bf8f9dba46bf483ed38df5fdd9b34f576f6 Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Thu, 25 May 2023 10:10:45 +0200 Subject: [PATCH 377/400] Update public README --- README.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.adoc b/README.adoc index 20d11acd34f..053e0c1d7c4 100644 --- a/README.adoc +++ b/README.adoc @@ -77,14 +77,14 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library .6+<.^|GDS 2.2.x |Neo4j 4.3.15 - 4.3.23 -.13+.^|Java 11 & Java 17 +.15+.^|Java 11 & Java 17 |Neo4j 4.4.9 - 4.4.16 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 |Neo4j 5.4.0 -.8+<.^|GDS 2.3.x -|Neo4j 4.4.9 - 4.4.20 +.9+<.^|GDS 2.3.x +|Neo4j 4.4.9 - 4.4.21 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 From 2e34d6332d30a979d7614f1bf454b860277016ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 25 May 2023 11:17:47 +0200 Subject: [PATCH 378/400] Throw on invalid negative ids Co-authored-by: Martin Junghanns --- .../gds/core/loading/LazyIdMapBuilder.java | 18 ++++++++++++++++++ .../gds/projection/CypherAggregationTest.java | 7 +++++++ 2 files changed, 25 insertions(+) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index 14ba2ca6124..05d92f93f17 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -29,6 +29,7 @@ import org.neo4j.gds.core.loading.construction.PropertyValues; import org.neo4j.gds.core.utils.paged.ShardedLongLongMap; +import java.util.Locale; import java.util.OptionalLong; import java.util.concurrent.atomic.AtomicBoolean; @@ -56,6 +57,8 @@ public void prepareForFlush() { public long addNode(long nodeId, NodeLabelToken nodeLabels) { long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); + checkPositiveId(nodeId); + // deduplication if (intermediateId < 0) { return -(intermediateId + 1); @@ -66,6 +69,19 @@ public long addNode(long nodeId, NodeLabelToken nodeLabels) { return intermediateId; } + /** + * GDS has the general assumption of non-negative original node ids. + */ + private static void checkPositiveId(long nodeId) { + if (nodeId < 0) { + throw new IllegalArgumentException(String.format( + Locale.US, + "GDS expects node ids to be positive. But got a negative id of `%d`.", + nodeId + )); + } + } + public long addNodeWithProperties( long nodeId, PropertyValues properties, @@ -73,6 +89,8 @@ public long addNodeWithProperties( ) { long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); + checkPositiveId(nodeId); + // deduplication if (intermediateId < 0) { return -(intermediateId + 1); diff --git a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java index 0ff2bd02465..3f997bb4afc 100644 --- a/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java +++ b/cypher-aggregation/src/test/java/org/neo4j/gds/projection/CypherAggregationTest.java @@ -218,6 +218,13 @@ void testInvalidArbitraryIds(String idLiteral, String invalidType) { .hasMessage("The node has to be either a NODE or an INTEGER, but got " + invalidType); } + @Test + void testInvalidNegativId() { + assertThatThrownBy(() -> runQuery("RETURN gds.alpha.graph.project('g', -1)")) + .rootCause() + .hasMessage("GDS expects node ids to be positive. But got a negative id of `-1`."); + } + @Test void testInvalidRelationshipAsArbitraryId() { var query = "MATCH ()-[r]-() RETURN gds.alpha.graph.project('g', r)"; From a337cd1fc5dad4e27cff13f30038d157d419fceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 25 May 2023 11:44:26 +0200 Subject: [PATCH 379/400] Move check before adding node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Max Kießling --- .../java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java index 05d92f93f17..f0c5597e1f6 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/LazyIdMapBuilder.java @@ -55,10 +55,10 @@ public void prepareForFlush() { } public long addNode(long nodeId, NodeLabelToken nodeLabels) { - long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); - checkPositiveId(nodeId); + long intermediateId = this.intermediateIdMapBuilder.addNode(nodeId); + // deduplication if (intermediateId < 0) { return -(intermediateId + 1); From b011f9c7314a58e568d7ce77cdd45d8856cd7d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 25 May 2023 16:28:59 +0200 Subject: [PATCH 380/400] Post 2.3.7+20 --- README.adoc | 14 +++++++------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index 053e0c1d7c4..256a5eb7360 100644 --- a/README.adoc +++ b/README.adoc @@ -148,21 +148,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.6 + 2.3.7. org.neo4j.gds algo - 2.3.6 + 2.3.7. org.neo4j.gds alpha-algo - 2.3.6 + 2.3.7. ---- @@ -174,28 +174,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.6 + 2.3.7. org.neo4j.gds proc - 2.3.6 + 2.3.7. org.neo4j.gds alpha-proc - 2.3.6 + 2.3.7. org.neo4j.gds write-services - 2.3.6 + 2.3.7. ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 545d7e209bc..0e0a3313280 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.7" +| "2.3.8" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index 970a31c4037..e1e63a62a31 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.7' + gdsVersion = '2.3.8' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index 55a819da2e7..6d22de93e7e 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { - gdsBaseVersion = '2.3.7' - gdsAuraVersion = '20' + gdsBaseVersion = '2.3.8' + gdsAuraVersion = '21' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 07e8571e86ac1230f616d71232936b1779616091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Mon, 22 May 2023 14:55:38 +0200 Subject: [PATCH 381/400] Add metric counting models created --- .../java/org/neo4j/gds/core/model/ModelCatalog.java | 12 +++++++++++- .../org/neo4j/gds/core/model/OpenModelCatalog.java | 11 +---------- .../neo4j/gds/core/model/OpenModelCatalogTest.java | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java index 729af8ba571..d0e5c5aa1ac 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java @@ -24,11 +24,21 @@ import org.neo4j.gds.model.ModelConfig; import java.util.Collection; +import java.util.HashSet; +import java.util.Set; import java.util.stream.Stream; public interface ModelCatalog { - void registerListener(ModelCatalogListener listener); + Set LISTENERS = new HashSet<>(); + + static void registerListener(ModelCatalogListener listener) { + LISTENERS.add(listener); + } + + static void unregisterListener(ModelCatalogListener listener) { + LISTENERS.remove(listener); + } void set(Model model); diff --git a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java index e5c6c587864..640204f7d83 100644 --- a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java +++ b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java @@ -26,7 +26,6 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; @@ -39,16 +38,8 @@ public final class OpenModelCatalog implements ModelCatalog { private final Map userCatalogs; - private final List listeners; - public OpenModelCatalog() { this.userCatalogs = new ConcurrentHashMap<>(); - this.listeners = new ArrayList<>(); - } - - @Override - public void registerListener(ModelCatalogListener listener) { - listeners.add(listener); } @Override @@ -61,7 +52,7 @@ public void set(Model model) { return userCatalog; }); - listeners.forEach(listener -> listener.onInsert(model)); + LISTENERS.forEach(listener -> listener.onInsert(model)); } @Override diff --git a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java index fc5b089699c..e8187158160 100644 --- a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java +++ b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java @@ -122,7 +122,7 @@ void shouldStoreModelsPerType() { @Test void shouldNotifyListeners() { var counter = new Counter(); - modelCatalog.registerListener(model -> counter.increment()); + ModelCatalog.registerListener(model -> counter.increment()); var model = Model.of( "testAlgo", From 2e77a8479e0545bf0d2991f76faa58f0365c99d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Thu, 25 May 2023 14:27:22 +0200 Subject: [PATCH 382/400] Reduce static state in metrics based on review comments by Mats Co-authored-by: Mats Rydberg --- .../org/neo4j/gds/core/model/ModelCatalog.java | 12 ++---------- .../neo4j/gds/core/model/OpenModelCatalog.java | 17 ++++++++++++++++- .../gds/core/model/OpenModelCatalogTest.java | 2 +- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java index d0e5c5aa1ac..3ccdeb79253 100644 --- a/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java +++ b/model-catalog-api/src/main/java/org/neo4j/gds/core/model/ModelCatalog.java @@ -24,21 +24,13 @@ import org.neo4j.gds.model.ModelConfig; import java.util.Collection; -import java.util.HashSet; -import java.util.Set; import java.util.stream.Stream; public interface ModelCatalog { - Set LISTENERS = new HashSet<>(); + void registerListener(ModelCatalogListener listener); - static void registerListener(ModelCatalogListener listener) { - LISTENERS.add(listener); - } - - static void unregisterListener(ModelCatalogListener listener) { - LISTENERS.remove(listener); - } + void unregisterListener(ModelCatalogListener listener); void set(Model model); diff --git a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java index 640204f7d83..6237e7a8cbf 100644 --- a/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java +++ b/open-model-catalog/src/main/java/org/neo4j/gds/core/model/OpenModelCatalog.java @@ -26,9 +26,11 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.Locale; import java.util.Map; import java.util.NoSuchElementException; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; @@ -38,8 +40,21 @@ public final class OpenModelCatalog implements ModelCatalog { private final Map userCatalogs; + private final Set listeners; + public OpenModelCatalog() { this.userCatalogs = new ConcurrentHashMap<>(); + this.listeners = new HashSet<>(); + } + + @Override + public void registerListener(ModelCatalogListener listener) { + listeners.add(listener); + } + + @Override + public void unregisterListener(ModelCatalogListener listener) { + listeners.remove(listener); } @Override @@ -52,7 +67,7 @@ public void set(Model model) { return userCatalog; }); - LISTENERS.forEach(listener -> listener.onInsert(model)); + listeners.forEach(listener -> listener.onInsert(model)); } @Override diff --git a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java index e8187158160..fc5b089699c 100644 --- a/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java +++ b/open-model-catalog/src/test/java/org/neo4j/gds/core/model/OpenModelCatalogTest.java @@ -122,7 +122,7 @@ void shouldStoreModelsPerType() { @Test void shouldNotifyListeners() { var counter = new Counter(); - ModelCatalog.registerListener(model -> counter.increment()); + modelCatalog.registerListener(model -> counter.increment()); var model = Model.of( "testAlgo", From 5cc78b6f0f155bd08cc26137bcd62e1632ff0909 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 30 May 2023 14:25:43 +0200 Subject: [PATCH 383/400] Exhibit issue --- .../UnionGraphWeightedNodeSimilarityTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java new file mode 100644 index 00000000000..a114b95e69b --- /dev/null +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.similarity.nodesim; + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; +import org.neo4j.gds.api.Graph; +import org.neo4j.gds.core.concurrency.Pools; +import org.neo4j.gds.core.utils.progress.tasks.ProgressTracker; +import org.neo4j.gds.extension.GdlExtension; +import org.neo4j.gds.extension.GdlGraph; +import org.neo4j.gds.extension.Inject; + +import static org.assertj.core.api.Assertions.assertThat; + +@GdlExtension + class UnionGraphWeightedNodeSimilarityTest { + + @GdlGraph + private static final String DB_CYPHER = + "CREATE " + + " (a1: A)," + + " (a2: A)," + + " (b1: B)," + + " (b2: B)," + + " (b3: B)," + + "(a1)-[:R1 {w: 5}]->(b1),"+ + "(a1)-[:R2 {w: 10}]->(b2),"+ + "(a2)-[:R2 {w: 10}]->(b2),"+ + "(a2)-[:R1 {w: 8}]->(b3)"; + + @Inject + private Graph graph; + + @Test + void shouldWorkWithUnionGraph(){ + var config=NodeSimilarityStreamConfigImpl.builder().relationshipWeightProperty("w").concurrency(1).topN(1).build(); + var nodeSimilarity=NodeSimilarity.create(graph,config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var result=nodeSimilarity.compute().streamResult().findFirst().get(); + //input should be (0 + 10 + 0)/ (5 + 10 + 8) = 10/23 + assertThat(result.similarity).isCloseTo((10)/(23.0), Offset.offset(1E-5)); + } +} From b05a046f3614ed5a60a9a707d73514c5b18bf830 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Tue, 30 May 2023 15:03:20 +0200 Subject: [PATCH 384/400] Fix bug --- .../similarity/nodesim/NodeSimilarity.java | 8 +- .../similarity/nodesim/VectorComputer.java | 17 +++- .../neo4j/gds/core/utils/TwoArraysSort.java | 79 +++++++++++++++++++ 3 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java index af069394e5d..00d7bcf40a6 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/NodeSimilarity.java @@ -33,7 +33,6 @@ import org.neo4j.gds.similarity.SimilarityResult; import org.neo4j.gds.similarity.filtering.NodeFilter; -import java.util.Arrays; import java.util.Comparator; import java.util.Objects; import java.util.Optional; @@ -217,12 +216,13 @@ private void prepare() { // TODO: we don't need to do the rest of the prepare for a node that isn't going to be used in the computation progressTracker.logProgress(graph.degree(node)); vectorComputer.forEachRelationship(node); + + if (sortVectors) { + vectorComputer.sortTargetIds(); + } if (weighted) { weights.set(node, vectorComputer.getWeights()); } - if (sortVectors) { - Arrays.sort(vectorComputer.targetIds.buffer); - } return vectorComputer.targetIds.buffer; } diff --git a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/VectorComputer.java b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/VectorComputer.java index bd48940112f..8c75e130e6c 100644 --- a/algo/src/main/java/org/neo4j/gds/similarity/nodesim/VectorComputer.java +++ b/algo/src/main/java/org/neo4j/gds/similarity/nodesim/VectorComputer.java @@ -25,9 +25,11 @@ import org.neo4j.gds.api.Graph; import org.neo4j.gds.api.RelationshipConsumer; import org.neo4j.gds.api.RelationshipWithPropertyConsumer; +import org.neo4j.gds.core.utils.TwoArraysSort; -import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; +import java.util.Arrays; +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; abstract class VectorComputer { final Graph graph; @@ -69,6 +71,8 @@ static VectorComputer of( : new UnweightedVectorComputer(graph); } + abstract void sortTargetIds(); + static final class UnweightedVectorComputer extends VectorComputer implements RelationshipConsumer { UnweightedVectorComputer(Graph graph) { @@ -91,6 +95,11 @@ public double[] getWeights() { )); } + @Override + void sortTargetIds() { + Arrays.sort(targetIds.buffer); + } + @Override void forEachRelationship(long node) { graph.forEachRelationship(node, this); @@ -129,5 +138,11 @@ void reset(int degree) { super.reset(degree); weights = new DoubleArrayList(degree, ARRAY_SIZING_STRATEGY); } + + @Override + void sortTargetIds() { + int length = weights.buffer.length; + TwoArraysSort.sortDoubleArrayByLongValues(targetIds.buffer, weights.buffer, length); + } } } diff --git a/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java b/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java new file mode 100644 index 00000000000..05c2b311b11 --- /dev/null +++ b/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.core.utils; + +import com.carrotsearch.hppc.sorting.IndirectComparator; +import com.carrotsearch.hppc.sorting.IndirectSort; + +public final class TwoArraysSort { + private TwoArraysSort() {} + + /** + * Sort two arrays simultaneously based on values of the first (long) array. + * E.g. {[4, 1, 8], [0.5, 1.9, 0.9]} -> {[1, 4, 8], [1,9, 0.5, 0.9]} + * + * @param longArray Array of long values (e.g. neighbours ids) + * @param doubleArray Array of double values (e.g. neighbours weighs) + * @param length Number of values to sort + */ + public static void sortDoubleArrayByLongValues(long[] longArray, double[] doubleArray, int length) { + assert longArray.length >= length; + assert doubleArray.length >= length; + var order = IndirectSort.mergesort(0, length, new AscendingLongComparator(longArray)); + reorder(order, longArray, doubleArray, length); + } + + private static void reorder(int[] order, long[] longArray, double[] doubleArray, int length) { + for (int i = 0; i < length; i++) { + while (order[i] != i) { + int idx = order[order[i]]; + var longVal = longArray[order[i]]; + var doubleVal = doubleArray[order[i]]; + + longArray[order[i]] = longArray[i]; + doubleArray[order[i]] = doubleArray[i]; + order[order[i]] = order[i]; + + order[i] = idx; + longArray[i] = longVal; + doubleArray[i] = doubleVal; + } + } + } + + public static class AscendingLongComparator implements IndirectComparator { + private final long[] array; + + AscendingLongComparator(long[] array) { + this.array = array; + } + + public int compare(int indexA, int indexB) { + final long a = array[indexA]; + final long b = array[indexB]; + + if (a < b) + return -1; + if (a > b) + return 1; + return 0; + } + } +} From e788b7f3f56176dc504f4f3b9ba66e7dff908c3f Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 31 May 2023 08:06:45 +0200 Subject: [PATCH 385/400] Don't create Neo4j paths for generated graphs. Co-authored-by: Veselin Nikolov --- .../ShortestPathStreamResultConsumer.java | 3 +- .../ShortestPathRandomGraphProcTest.java | 65 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathRandomGraphProcTest.java diff --git a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java index bb5a9635091..cf571b1fd4a 100644 --- a/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java +++ b/proc/path-finding/src/main/java/org/neo4j/gds/paths/ShortestPathStreamResultConsumer.java @@ -45,7 +45,8 @@ public Stream consume( var shouldReturnPath = executionContext.callContext() .outputFields() - .anyMatch(field -> toLowerCaseWithLocale(field).equals("path")); + .anyMatch(field -> toLowerCaseWithLocale(field).equals("path")) + && computationResult.graphStore().capabilities().canWriteToDatabase(); var resultBuilder = new StreamResult.Builder(graph, executionContext.transaction().internalTransaction()); diff --git a/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathRandomGraphProcTest.java b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathRandomGraphProcTest.java new file mode 100644 index 00000000000..d0e76eec189 --- /dev/null +++ b/proc/path-finding/src/test/java/org/neo4j/gds/paths/sourcetarget/ShortestPathRandomGraphProcTest.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.paths.sourcetarget; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.neo4j.gds.BaseProcTest; +import org.neo4j.gds.beta.generator.GraphGenerateProc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatNoException; + + class ShortestPathRandomGraphProcTest extends BaseProcTest { + + @BeforeEach + void setup() throws Exception { + registerProcedures( + ShortestPathYensStreamProc.class, + ShortestPathDijkstraStreamProc.class, + ShortestPathAStarStreamProc.class, + GraphGenerateProc.class + ); + } + + @ParameterizedTest + @ValueSource( + strings = { + "CALL gds.shortestPath.yens.stream('graph',{k:1, sourceNode:0, targetNode:1})", + "CALL gds.shortestPath.dijkstra.stream('graph',{sourceNode:0, targetNode:1})" + } + ) + void shouldWorkWithRandomGraph(String query) { + + runQuery("CALL gds.beta.graph.generate('graph',10,10)"); + assertThatNoException().isThrownBy(() -> { + + long rowCount = runQueryWithRowConsumer(query, result -> { + assertThat(result.get("path")).isNull(); + }); + assertThat(rowCount).isEqualTo(1L); + + }); + + + } + +} From 91a3891d0fc71596ff9eb6217711dedcaa80aa68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Wed, 31 May 2023 09:09:08 +0200 Subject: [PATCH 386/400] Update 2.3 supported database versions --- .../ROOT/pages/installation/supported-neo4j-versions.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc index 2e567fb174c..ef221582a62 100644 --- a/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc +++ b/doc/modules/ROOT/pages/installation/supported-neo4j-versions.adoc @@ -10,6 +10,8 @@ Time to upgrade! [opts=header] |=== | Neo4j version | Neo4j Graph Data Science +| `5.8` | `2.3.6 or later` +| `5.7` | `2.3.3 or later` | `5.6` | `2.3.2 or later` | `5.5` | `2.3.1 or later` | `5.4` | `2.3`, `2.2.7 or later` footnote:eol[This version series is end-of-life and will not receive further patches. Please use a later version.] From 60e0a28532d64805cc4900cc90e0dd2d1678c461 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 31 May 2023 11:15:31 +0200 Subject: [PATCH 387/400] Whitespace --- .../UnionGraphWeightedNodeSimilarityTest.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java index a114b95e69b..0c4ab250cad 100644 --- a/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java +++ b/algo/src/test/java/org/neo4j/gds/similarity/nodesim/UnionGraphWeightedNodeSimilarityTest.java @@ -51,10 +51,17 @@ class UnionGraphWeightedNodeSimilarityTest { @Test void shouldWorkWithUnionGraph(){ - var config=NodeSimilarityStreamConfigImpl.builder().relationshipWeightProperty("w").concurrency(1).topN(1).build(); - var nodeSimilarity=NodeSimilarity.create(graph,config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); - var result=nodeSimilarity.compute().streamResult().findFirst().get(); + + var config = NodeSimilarityStreamConfigImpl.builder() + .relationshipWeightProperty("w") + .concurrency(1) + .topN(1) + .build(); + + var nodeSimilarity = NodeSimilarity.create(graph, config, Pools.DEFAULT, ProgressTracker.NULL_TRACKER); + var result = nodeSimilarity.compute().streamResult().findFirst().get(); + //input should be (0 + 10 + 0)/ (5 + 10 + 8) = 10/23 - assertThat(result.similarity).isCloseTo((10)/(23.0), Offset.offset(1E-5)); + assertThat(result.similarity).isCloseTo((10) / (23.0), Offset.offset(1E-5)); } } From 9a1f85dc241142716920a6ebbbb89a2cfa76659a Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 31 May 2023 14:52:14 +0200 Subject: [PATCH 388/400] something something javadoc --- core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java b/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java index 05c2b311b11..94b91772f8f 100644 --- a/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java +++ b/core/src/main/java/org/neo4j/gds/core/utils/TwoArraysSort.java @@ -24,10 +24,9 @@ public final class TwoArraysSort { private TwoArraysSort() {} - /** * Sort two arrays simultaneously based on values of the first (long) array. - * E.g. {[4, 1, 8], [0.5, 1.9, 0.9]} -> {[1, 4, 8], [1,9, 0.5, 0.9]} + * E.g. {[4, 1, 8], [0.5, 1.9, 0.9]} yields {[1, 4, 8], [1,9, 0.5, 0.9]} * * @param longArray Array of long values (e.g. neighbours ids) * @param doubleArray Array of double values (e.g. neighbours weighs) From a0575cc460247973bf4e5ec9222e858cdf84b6bc Mon Sep 17 00:00:00 2001 From: ioannispan Date: Thu, 1 Jun 2023 16:56:50 +0200 Subject: [PATCH 389/400] Post release actions Co-authored-by: Veselin Nikolov --- README.adoc | 14 +++++++------- .../pages/management-ops/utility-functions.adoc | 2 +- examples/pregel-bootstrap/build.gradle | 2 +- gradle/version.gradle | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.adoc b/README.adoc index 256a5eb7360..e8b9671cc02 100644 --- a/README.adoc +++ b/README.adoc @@ -148,21 +148,21 @@ The algorithms are located in the `algo-common`, `algo` and `alpha-algo` modules org.neo4j.gds algo-common - 2.3.7. + 2.3.8 org.neo4j.gds algo - 2.3.7. + 2.3.8 org.neo4j.gds alpha-algo - 2.3.7. + 2.3.8 ---- @@ -174,28 +174,28 @@ The procedures are located in the `proc-common`, `proc` and `alpha-proc` modules org.neo4j.gds proc-common - 2.3.7. + 2.3.8 org.neo4j.gds proc - 2.3.7. + 2.3.8 org.neo4j.gds alpha-proc - 2.3.7. + 2.3.8 org.neo4j.gds write-services - 2.3.7. + 2.3.8 ---- diff --git a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc index 0e0a3313280..d80ad724319 100644 --- a/doc/modules/ROOT/pages/management-ops/utility-functions.adoc +++ b/doc/modules/ROOT/pages/management-ops/utility-functions.adoc @@ -27,7 +27,7 @@ RETURN gds.version() AS version [opts="header"] |=== | version -| "2.3.8" +| "2.3.9" |=== -- diff --git a/examples/pregel-bootstrap/build.gradle b/examples/pregel-bootstrap/build.gradle index e1e63a62a31..2a1996a0af3 100644 --- a/examples/pregel-bootstrap/build.gradle +++ b/examples/pregel-bootstrap/build.gradle @@ -7,7 +7,7 @@ plugins { ext { // Make sure these are the same as your installation of GDS and Neo4j - gdsVersion = '2.3.8' + gdsVersion = '2.3.9' neo4jVersion = '5.1.0' // Necessary to generate value classes for Pregel configs diff --git a/gradle/version.gradle b/gradle/version.gradle index 6d22de93e7e..929fbf0acdd 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { - gdsBaseVersion = '2.3.8' - gdsAuraVersion = '21' + gdsBaseVersion = '2.3.9' + gdsAuraVersion = '22' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From d1b28e06b79957f3b3d8abc3e9d05340035576c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Mon, 5 Jun 2023 12:43:53 +0200 Subject: [PATCH 390/400] Public 5.9 compat --- .../5.9/neo4j-kernel-adapter/build.gradle | 63 ++ .../gds/compat/_59/Neo4jProxyFactoryImpl.java | 44 + .../compat/_59/SettingProxyFactoryImpl.java | 44 + .../compat/_59/BoltTransactionRunnerImpl.java | 97 ++ .../gds/compat/_59/CallableProcedureImpl.java | 53 + .../CallableUserAggregationFunctionImpl.java | 77 ++ .../gds/compat/_59/CompatAccessModeImpl.java | 44 + .../_59/CompatGraphDatabaseAPIImpl.java | 74 ++ .../gds/compat/_59/CompatIndexQueryImpl.java | 31 + .../_59/CompatUsernameAuthSubjectImpl.java | 35 + .../compat/_59/CompositeNodeCursorImpl.java | 32 + .../gds/compat/_59/GdsDatabaseLayoutImpl.java | 50 + ...sDatabaseManagementServiceBuilderImpl.java | 54 + .../gds/compat/_59/Neo4jProxyFactoryImpl.java | 44 + .../neo4j/gds/compat/_59/Neo4jProxyImpl.java | 930 ++++++++++++++++++ .../compat/_59/NodeLabelIndexLookupImpl.java | 68 ++ .../gds/compat/_59/PartitionedStoreScan.java | 58 ++ .../_59/ReferencePropertyReference.java | 49 + .../compat/_59/ScanBasedStoreScanImpl.java | 40 + .../compat/_59/SettingProxyFactoryImpl.java | 44 + .../gds/compat/_59/SettingProxyImpl.java | 87 ++ .../org/neo4j/gds/compat/_59/TestLogImpl.java | 146 +++ .../compat/_59/VirtualRelationshipImpl.java | 41 + .../5.9/storage-engine-adapter/build.gradle | 66 ++ .../_59/InMemoryStorageEngineFactory.java | 268 +++++ .../_59/StorageEngineProxyFactoryImpl.java | 44 + .../InMemoryCommandCreationContextImpl.java | 107 ++ .../compat/_59/InMemoryCountsStoreImpl.java | 112 +++ .../_59/InMemoryMetaDataProviderImpl.java | 201 ++++ .../gds/compat/_59/InMemoryNodeCursor.java | 82 ++ .../_59/InMemoryNodePropertyCursor.java | 45 + .../compat/_59/InMemoryPropertyCursor.java | 71 ++ .../_59/InMemoryPropertySelectionImpl.java | 55 ++ .../InMemoryRelationshipPropertyCursor.java | 60 ++ .../_59/InMemoryRelationshipScanCursor.java | 61 ++ .../InMemoryRelationshipTraversalCursor.java | 47 + .../_59/InMemoryStorageEngineFactory.java | 558 +++++++++++ .../compat/_59/InMemoryStorageEngineImpl.java | 341 +++++++ .../compat/_59/InMemoryStorageLocksImpl.java | 86 ++ .../gds/compat/_59/InMemoryStoreVersion.java | 68 ++ .../_59/InMemoryTransactionIdStoreImpl.java | 117 +++ .../gds/compat/_59/InMemoryVersionCheck.java | 61 ++ .../_59/StorageEngineProxyFactoryImpl.java | 44 + .../compat/_59/StorageEngineProxyImpl.java | 156 +++ .../InMemoryLogVersionRepository59.java | 71 ++ ...InMemoryStorageCommandReaderFactory59.java | 43 + .../InMemoryStorageReader59.java | 332 +++++++ gradle/dependencies.gradle | 1 + .../org/neo4j/gds/compat/Neo4jVersion.java | 7 +- .../neo4j/gds/compat/Neo4jVersionTest.java | 4 +- .../java/org/neo4j/gds/SysInfoProcTest.java | 13 + 51 files changed, 5324 insertions(+), 2 deletions(-) create mode 100644 compatibility/5.9/neo4j-kernel-adapter/build.gradle create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java create mode 100644 compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/build.gradle create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java create mode 100644 compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java diff --git a/compatibility/5.9/neo4j-kernel-adapter/build.gradle b/compatibility/5.9/neo4j-kernel-adapter/build.gradle new file mode 100644 index 00000000000..e688f9dcf48 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/build.gradle @@ -0,0 +1,63 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Neo4j Kernel Adapter 5.9' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + compileOnly project(':annotations') + compileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + compileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.9' + + implementation project(':neo4j-kernel-adapter-api') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + + implementation project(':neo4j-kernel-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j.community', name: 'it-test-support', version: neos.'5.9' + java17CompileOnly group: 'com.github.spotbugs', name: 'spotbugs-annotations', version: ver.'spotbugsToolVersion' + + java17Implementation project(':neo4j-kernel-adapter-api') + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..c972ffc94a3 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public Neo4jProxyApi load() { + throw new UnsupportedOperationException("5.9 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j 5.9 (placeholder)"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..1a05f4b02fd --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public SettingProxyApi load() { + throw new UnsupportedOperationException("5.9 compatibility requires JDK17"); + } + + @Override + public String description() { + return "Neo4j Settings 5.9 (placeholder)"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java new file mode 100644 index 00000000000..479f8dbf179 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/BoltTransactionRunnerImpl.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.bolt.dbapi.BoltGraphDatabaseServiceSPI; +import org.neo4j.bolt.dbapi.BoltTransaction; +import org.neo4j.bolt.protocol.common.message.AccessMode; +import org.neo4j.bolt.protocol.common.message.request.connection.RoutingContext; +import org.neo4j.bolt.tx.statement.StatementQuerySubscriber; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.BoltQuerySubscriber; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.graphdb.QueryStatistics; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.QueryExecutionKernelException; +import org.neo4j.values.virtual.MapValue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +public class BoltTransactionRunnerImpl extends BoltTransactionRunner { + + @Override + protected BoltQuerySubscriber boltQuerySubscriber() { + var subscriber = new StatementQuerySubscriber(); + return new BoltQuerySubscriber<>() { + @Override + public void assertSucceeded() throws KernelException { + subscriber.assertSuccess(); + } + + @Override + public QueryStatistics queryStatistics() { + return subscriber.getStatistics(); + } + + @Override + public StatementQuerySubscriber innerSubscriber() { + return subscriber; + } + }; + } + + @Override + protected void executeQuery( + BoltTransaction boltTransaction, + String query, + MapValue parameters, + StatementQuerySubscriber querySubscriber + ) throws QueryExecutionKernelException { + boltTransaction.executeQuery(query, parameters, true, querySubscriber); + } + + @Override + protected BoltTransaction beginBoltWriteTransaction( + BoltGraphDatabaseServiceSPI fabricDb, + LoginContext loginContext, + KernelTransaction.Type kernelTransactionType, + ClientConnectionInfo clientConnectionInfo, + List bookmarks, + Duration txTimeout, + Map txMetadata + ) { + return fabricDb.beginTransaction( + kernelTransactionType, + loginContext, + clientConnectionInfo, + bookmarks, + txTimeout, + AccessMode.WRITE, + txMetadata, + new RoutingContext(true, Map.of()), + QueryExecutionConfiguration.DEFAULT_CONFIG + ); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java new file mode 100644 index 00000000000..08ff067f966 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableProcedureImpl.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.collection.RawIterator; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.kernel.api.ResourceMonitor; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableProcedureImpl implements CallableProcedure { + private final CompatCallableProcedure procedure; + + CallableProcedureImpl(CompatCallableProcedure procedure) { + this.procedure = procedure; + } + + @Override + public ProcedureSignature signature() { + return this.procedure.signature(); + } + + @Override + public RawIterator apply( + Context ctx, + AnyValue[] input, + ResourceMonitor resourceMonitor + ) throws ProcedureException { + return this.procedure.apply(ctx, input); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java new file mode 100644 index 00000000000..de837dc4b2e --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CallableUserAggregationFunctionImpl.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompatUserAggregator; +import org.neo4j.internal.kernel.api.exceptions.ProcedureException; +import org.neo4j.internal.kernel.api.procs.UserAggregationReducer; +import org.neo4j.internal.kernel.api.procs.UserAggregationUpdater; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.api.procedure.Context; +import org.neo4j.values.AnyValue; + +@SuppressForbidden(reason = "This is the compat API") +public final class CallableUserAggregationFunctionImpl implements CallableUserAggregationFunction { + private final CompatUserAggregationFunction function; + + CallableUserAggregationFunctionImpl(CompatUserAggregationFunction function) { + this.function = function; + } + + @Override + public UserFunctionSignature signature() { + return this.function.signature(); + } + + @Override + public UserAggregationReducer createReducer(Context ctx) throws ProcedureException { + return new UserAggregatorImpl(this.function.create(ctx)); + } + + private static final class UserAggregatorImpl implements UserAggregationReducer, UserAggregationUpdater { + private final CompatUserAggregator aggregator; + + private UserAggregatorImpl(CompatUserAggregator aggregator) { + this.aggregator = aggregator; + } + + @Override + public UserAggregationUpdater newUpdater() { + return this; + } + + @Override + public void update(AnyValue[] input) throws ProcedureException { + this.aggregator.update(input); + } + + @Override + public void applyUpdates() { + } + + @Override + public AnyValue result() throws ProcedureException { + return this.aggregator.result(); + } + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java new file mode 100644 index 00000000000..49f92fb86e5 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatAccessModeImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatAccessMode; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.internal.kernel.api.RelTypeSupplier; +import org.neo4j.internal.kernel.api.TokenSet; + +import java.util.function.Supplier; + +public final class CompatAccessModeImpl extends CompatAccessMode { + + CompatAccessModeImpl(CustomAccessMode custom) { + super(custom); + } + + @Override + public boolean allowsReadNodeProperty(Supplier labels, int propertyKey) { + return custom.allowsReadNodeProperty(propertyKey); + } + + @Override + public boolean allowsReadRelationshipProperty(RelTypeSupplier relType, int propertyKey) { + return custom.allowsReadRelationshipProperty(propertyKey); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java new file mode 100644 index 00000000000..4fbf4bafb57 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatGraphDatabaseAPIImpl.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.security.LoginContext; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.coreapi.TransactionExceptionMapper; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +final class CompatGraphDatabaseAPIImpl extends GdsGraphDatabaseAPI { + + CompatGraphDatabaseAPIImpl(DatabaseManagementService dbms) { + super(dbms); + } + + @Override + public boolean isAvailable() { + return api.isAvailable(); + } + + @Override + public TopologyGraphDbmsModel.HostedOnMode mode() { + // NOTE: This means we can never start clusters locally, which is probably fine since: + // 1) We never did this before + // 2) We only use this for tests and benchmarks + return TopologyGraphDbmsModel.HostedOnMode.SINGLE; + } + + @Override + public InternalTransaction beginTransaction( + KernelTransaction.Type type, + LoginContext loginContext, + ClientConnectionInfo clientInfo, + long timeout, + TimeUnit unit, + Consumer terminationCallback, + TransactionExceptionMapper transactionExceptionMapper + ) { + return api.beginTransaction( + type, + loginContext, + clientInfo, + timeout, + unit, + terminationCallback, + transactionExceptionMapper + ); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java new file mode 100644 index 00000000000..ccef737c591 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatIndexQueryImpl.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; + +final class CompatIndexQueryImpl implements CompatIndexQuery { + final PropertyIndexQuery indexQuery; + + CompatIndexQueryImpl(PropertyIndexQuery indexQuery) { + this.indexQuery = indexQuery; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java new file mode 100644 index 00000000000..85c75a5d377 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompatUsernameAuthSubjectImpl.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompatUsernameAuthSubject; +import org.neo4j.internal.kernel.api.security.AuthSubject; + +final class CompatUsernameAuthSubjectImpl extends CompatUsernameAuthSubject { + + CompatUsernameAuthSubjectImpl(String username, AuthSubject authSubject) { + super(username, authSubject); + } + + @Override + public String executingUser() { + return username; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java new file mode 100644 index 00000000000..3fc8fbfca62 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/CompositeNodeCursorImpl.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; + +import java.util.List; + +public final class CompositeNodeCursorImpl extends CompositeNodeCursor { + + CompositeNodeCursorImpl(List cursors, int[] labelIds) { + super(cursors, labelIds); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java new file mode 100644 index 00000000000..01a2134ad14 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseLayoutImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.io.layout.DatabaseLayout; + +import java.nio.file.Path; + +public class GdsDatabaseLayoutImpl implements GdsDatabaseLayout { + private final DatabaseLayout databaseLayout; + + public GdsDatabaseLayoutImpl(DatabaseLayout databaseLayout) {this.databaseLayout = databaseLayout;} + + @Override + public Path databaseDirectory() { + return databaseLayout.databaseDirectory(); + } + + @Override + public Path getTransactionLogsDirectory() { + return databaseLayout.getTransactionLogsDirectory(); + } + + @Override + public Path metadataStore() { + return databaseLayout.metadataStore(); + } + + public DatabaseLayout databaseLayout() { + return databaseLayout; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java new file mode 100644 index 00000000000..e8e8185129c --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/GdsDatabaseManagementServiceBuilderImpl.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.dbms.api.DatabaseManagementServiceBuilderImplementation; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.graphdb.config.Setting; + +import java.nio.file.Path; +import java.util.Map; + +public class GdsDatabaseManagementServiceBuilderImpl implements GdsDatabaseManagementServiceBuilder { + + private final DatabaseManagementServiceBuilderImplementation dbmsBuilder; + + GdsDatabaseManagementServiceBuilderImpl(Path storeDir) { + this.dbmsBuilder = new DatabaseManagementServiceBuilderImplementation(storeDir); + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfigRaw(Map configMap) { + dbmsBuilder.setConfigRaw(configMap); + return this; + } + + @Override + public GdsDatabaseManagementServiceBuilder setConfig(Setting setting, S value) { + dbmsBuilder.setConfig(setting, value); + return this; + } + + @Override + public DatabaseManagementService build() { + return dbmsBuilder.build(); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java new file mode 100644 index 00000000000..25ad1de6d4e --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.Neo4jProxyFactory; +import org.neo4j.gds.compat.Neo4jVersion; + +@ServiceProvider +public final class Neo4jProxyFactoryImpl implements Neo4jProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public Neo4jProxyApi load() { + return new Neo4jProxyImpl(); + } + + @Override + public String description() { + return "Neo4j 5.9"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java new file mode 100644 index 00000000000..f87ab180270 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/Neo4jProxyImpl.java @@ -0,0 +1,930 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.neo4j.common.DependencyResolver; +import org.neo4j.common.EntityType; +import org.neo4j.configuration.BootloaderSettings; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseSettings; +import org.neo4j.configuration.SettingValueParsers; +import org.neo4j.configuration.connectors.ConnectorPortRegister; +import org.neo4j.configuration.connectors.ConnectorType; +import org.neo4j.configuration.helpers.DatabaseNameValidator; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.exceptions.KernelException; +import org.neo4j.fabric.FabricDatabaseManager; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.BoltTransactionRunner; +import org.neo4j.gds.compat.CompatCallableProcedure; +import org.neo4j.gds.compat.CompatExecutionMonitor; +import org.neo4j.gds.compat.CompatIndexQuery; +import org.neo4j.gds.compat.CompatInput; +import org.neo4j.gds.compat.CompatUserAggregationFunction; +import org.neo4j.gds.compat.CompositeNodeCursor; +import org.neo4j.gds.compat.CustomAccessMode; +import org.neo4j.gds.compat.GdsDatabaseLayout; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GdsGraphDatabaseAPI; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.InputEntityIdVisitor; +import org.neo4j.gds.compat.Neo4jProxyApi; +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.gds.compat.TestLog; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.BatchImporterFactory; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IndexConfig; +import org.neo4j.internal.batchimport.InputIterable; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.IdType; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.InputEntityVisitor; +import org.neo4j.internal.batchimport.input.PropertySizeCalculator; +import org.neo4j.internal.batchimport.input.ReadableGroups; +import org.neo4j.internal.batchimport.staging.ExecutionMonitor; +import org.neo4j.internal.batchimport.staging.StageExecution; +import org.neo4j.internal.helpers.HostnamePort; +import org.neo4j.internal.id.IdGenerator; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.IndexQueryConstraints; +import org.neo4j.internal.kernel.api.IndexReadSession; +import org.neo4j.internal.kernel.api.NodeCursor; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.NodeValueIndexCursor; +import org.neo4j.internal.kernel.api.PropertyCursor; +import org.neo4j.internal.kernel.api.PropertyIndexQuery; +import org.neo4j.internal.kernel.api.QueryContext; +import org.neo4j.internal.kernel.api.Read; +import org.neo4j.internal.kernel.api.RelationshipScanCursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.internal.kernel.api.TokenPredicate; +import org.neo4j.internal.kernel.api.connectioninfo.ClientConnectionInfo; +import org.neo4j.internal.kernel.api.procs.FieldSignature; +import org.neo4j.internal.kernel.api.procs.Neo4jTypes; +import org.neo4j.internal.kernel.api.procs.ProcedureSignature; +import org.neo4j.internal.kernel.api.procs.QualifiedName; +import org.neo4j.internal.kernel.api.procs.UserFunctionSignature; +import org.neo4j.internal.kernel.api.security.AccessMode; +import org.neo4j.internal.kernel.api.security.AuthSubject; +import org.neo4j.internal.kernel.api.security.SecurityContext; +import org.neo4j.internal.recordstorage.RecordIdType; +import org.neo4j.internal.schema.IndexCapability; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexOrder; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.context.EmptyVersionContextSupplier; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.api.KernelTransaction; +import org.neo4j.kernel.api.KernelTransactionHandle; +import org.neo4j.kernel.api.procedure.CallableProcedure; +import org.neo4j.kernel.api.procedure.CallableUserAggregationFunction; +import org.neo4j.kernel.database.NamedDatabaseId; +import org.neo4j.kernel.database.NormalizedDatabaseName; +import org.neo4j.kernel.database.TestDatabaseIdRepository; +import org.neo4j.kernel.impl.coreapi.InternalTransaction; +import org.neo4j.kernel.impl.index.schema.IndexImporterFactoryImpl; +import org.neo4j.kernel.impl.query.QueryExecutionConfiguration; +import org.neo4j.kernel.impl.query.TransactionalContext; +import org.neo4j.kernel.impl.query.TransactionalContextFactory; +import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.format.RecordFormatSelector; +import org.neo4j.kernel.impl.store.format.RecordFormats; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.transaction.log.EmptyLogTailMetadata; +import org.neo4j.kernel.impl.transaction.log.files.TransactionLogInitializer; +import org.neo4j.logging.Log; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.EmptyMemoryTracker; +import org.neo4j.procedure.Mode; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.ssl.config.SslPolicyLoader; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.util.Bits; +import org.neo4j.values.storable.ValueCategory; +import org.neo4j.values.storable.Values; +import org.neo4j.values.virtual.MapValue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.neo4j.gds.compat.InternalReadOps.countByIdGenerator; +import static org.neo4j.io.pagecache.context.EmptyVersionContextSupplier.EMPTY; + +public final class Neo4jProxyImpl implements Neo4jProxyApi { + + @Override + public GdsGraphDatabaseAPI newDb(DatabaseManagementService dbms) { + return new CompatGraphDatabaseAPIImpl(dbms); + } + + @Override + public String validateExternalDatabaseName(String databaseName) { + var normalizedName = new NormalizedDatabaseName(databaseName); + DatabaseNameValidator.validateExternalDatabaseName(normalizedName); + return normalizedName.name(); + } + + @Override + public AccessMode accessMode(CustomAccessMode customAccessMode) { + return new CompatAccessModeImpl(customAccessMode); + } + + @Override + public String username(AuthSubject subject) { + return subject.executingUser(); + } + + @Override + public SecurityContext securityContext( + String username, + AuthSubject authSubject, + AccessMode mode, + String databaseName + ) { + return new SecurityContext( + new CompatUsernameAuthSubjectImpl(username, authSubject), + mode, + // GDS is always operating from an embedded context + ClientConnectionInfo.EMBEDDED_CONNECTION, + databaseName + ); + } + + @Override + public long getHighestPossibleIdInUse( + RecordStore recordStore, + KernelTransaction kernelTransaction + ) { + return recordStore.getHighestPossibleIdInUse(kernelTransaction.cursorContext()); + } + + @Override + public long getHighId(RecordStore recordStore) { + return recordStore.getIdGenerator().getHighId(); + } + + @Override + public List> entityCursorScan( + KernelTransaction transaction, + int[] labelIds, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelIds); + } else { + var read = transaction.dataRead(); + return Arrays + .stream(labelIds) + .mapToObj(read::nodeLabelScan) + .map(scan -> scanToStoreScan(scan, batchSize)) + .collect(Collectors.toList()); + } + } + + @Override + public PropertyCursor allocatePropertyCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocatePropertyCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public PropertyReference propertyReference(NodeCursor nodeCursor) { + return ReferencePropertyReference.of(nodeCursor.propertiesReference()); + } + + @Override + public PropertyReference propertyReference(RelationshipScanCursor relationshipScanCursor) { + return ReferencePropertyReference.of(relationshipScanCursor.propertiesReference()); + } + + @Override + public PropertyReference noPropertyReference() { + return ReferencePropertyReference.empty(); + } + + @Override + public void nodeProperties( + KernelTransaction kernelTransaction, + long nodeReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .nodeProperties(nodeReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public void relationshipProperties( + KernelTransaction kernelTransaction, + long relationshipReference, + PropertyReference reference, + PropertyCursor cursor + ) { + var neoReference = ((ReferencePropertyReference) reference).reference; + kernelTransaction + .dataRead() + .relationshipProperties(relationshipReference, neoReference, PropertySelection.ALL_PROPERTIES, cursor); + } + + @Override + public NodeCursor allocateNodeCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeCursor(kernelTransaction.cursorContext()); + } + + @Override + public RelationshipScanCursor allocateRelationshipScanCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateRelationshipScanCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeLabelIndexCursor allocateNodeLabelIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction.cursors().allocateNodeLabelIndexCursor(kernelTransaction.cursorContext()); + } + + @Override + public NodeValueIndexCursor allocateNodeValueIndexCursor(KernelTransaction kernelTransaction) { + return kernelTransaction + .cursors() + .allocateNodeValueIndexCursor(kernelTransaction.cursorContext(), kernelTransaction.memoryTracker()); + } + + @Override + public boolean hasNodeLabelIndex(KernelTransaction kernelTransaction) { + return NodeLabelIndexLookupImpl.hasNodeLabelIndex(kernelTransaction); + } + + @Override + public StoreScan nodeLabelIndexScan( + KernelTransaction transaction, + int labelId, + int batchSize, + boolean allowPartitionedScan + ) { + if (allowPartitionedScan) { + return partitionedNodeLabelIndexScan(transaction, batchSize, labelId).get(0); + } else { + var read = transaction.dataRead(); + return scanToStoreScan(read.nodeLabelScan(labelId), batchSize); + } + } + + @Override + public StoreScan scanToStoreScan(Scan scan, int batchSize) { + return new ScanBasedStoreScanImpl<>(scan, batchSize); + } + + private List> partitionedNodeLabelIndexScan( + KernelTransaction transaction, + int batchSize, + int... labelIds + ) { + var indexDescriptor = NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ); + + if (indexDescriptor == IndexDescriptor.NO_INDEX) { + throw new IllegalStateException("There is no index that can back a node label scan."); + } + + var read = transaction.dataRead(); + + // Our current strategy is to select the token with the highest count + // and use that one as the driving partitioned index scan. The partitions + // of all other partitioned index scans will be aligned to that one. + int maxToken = labelIds[0]; + long maxCount = read.countsForNodeWithoutTxState(labelIds[0]); + + for (int i = 1; i < labelIds.length; i++) { + long count = read.countsForNodeWithoutTxState(labelIds[i]); + if (count > maxCount) { + maxCount = count; + maxToken = labelIds[i]; + } + } + + int numberOfPartitions = PartitionedStoreScan.getNumberOfPartitions(maxCount, batchSize); + + try { + var session = read.tokenReadSession(indexDescriptor); + + var partitionedScan = read.nodeLabelScan( + session, + numberOfPartitions, + transaction.cursorContext(), + new TokenPredicate(maxToken) + ); + + var scans = new ArrayList>(labelIds.length); + scans.add(new PartitionedStoreScan(partitionedScan)); + + // Initialize the remaining index scans with the partitioning of the first scan. + for (int labelToken : labelIds) { + if (labelToken != maxToken) { + var scan = read.nodeLabelScan(session, partitionedScan, new TokenPredicate(labelToken)); + scans.add(new PartitionedStoreScan(scan)); + } + } + + return scans; + } catch (KernelException e) { + // should not happen, we check for the index existence and applicability + // before reading it + throw new RuntimeException("Unexpected error while initialising reading from node label index", e); + } + } + + @Override + public CompatIndexQuery rangeIndexQuery( + int propertyKeyId, + double from, + boolean fromInclusive, + double to, + boolean toInclusive + ) { + return new CompatIndexQueryImpl(PropertyIndexQuery.range(propertyKeyId, from, fromInclusive, to, toInclusive)); + } + + @Override + public CompatIndexQuery rangeAllIndexQuery(int propertyKeyId) { + var rangePredicate = PropertyIndexQuery.range( + propertyKeyId, + Values.doubleValue(Double.NEGATIVE_INFINITY), + true, + Values.doubleValue(Double.POSITIVE_INFINITY), + true + ); + return new CompatIndexQueryImpl(rangePredicate); + } + + @Override + public void nodeIndexSeek( + Read dataRead, + IndexReadSession index, + NodeValueIndexCursor cursor, + IndexOrder indexOrder, + boolean needsValues, + CompatIndexQuery query + ) throws KernelException { + var indexQueryConstraints = indexOrder == IndexOrder.NONE + ? IndexQueryConstraints.unordered(needsValues) + : IndexQueryConstraints.constrained(indexOrder, needsValues); + + dataRead.nodeIndexSeek( + (QueryContext) dataRead, + index, + cursor, + indexQueryConstraints, + ((CompatIndexQueryImpl) query).indexQuery + ); + } + + @Override + public CompositeNodeCursor compositeNodeCursor(List cursors, int[] labelIds) { + return new CompositeNodeCursorImpl(cursors, labelIds); + } + + @Override + public Configuration batchImporterConfig( + int batchSize, + int writeConcurrency, + Optional pageCacheMemory, + boolean highIO, + IndexConfig indexConfig + ) { + return new org.neo4j.internal.batchimport.Configuration() { + @Override + public int batchSize() { + return batchSize; + } + + @Override + public int maxNumberOfWorkerThreads() { + return writeConcurrency; + } + + @Override + public boolean highIO() { + return highIO; + } + + @Override + public IndexConfig indexConfig() { + return indexConfig; + } + }; + } + + @Override + public int writeConcurrency(Configuration batchImportConfiguration) { + return batchImportConfiguration.maxNumberOfWorkerThreads(); + } + + @Override + public BatchImporter instantiateBatchImporter( + BatchImporterFactory factory, + GdsDatabaseLayout directoryStructure, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + ExecutionMonitor executionMonitor, + AdditionalInitialIds additionalInitialIds, + Config dbConfig, + RecordFormats recordFormats, + JobScheduler jobScheduler, + Collector badCollector + ) { + dbConfig.set(GraphDatabaseSettings.db_format, recordFormats.name()); + var databaseLayout = ((GdsDatabaseLayoutImpl) directoryStructure).databaseLayout(); + return factory.instantiate( + databaseLayout, + fileSystem, + pageCacheTracer, + configuration, + logService, + executionMonitor, + additionalInitialIds, + new EmptyLogTailMetadata(dbConfig), + dbConfig, + Monitor.NO_MONITOR, + jobScheduler, + badCollector, + TransactionLogInitializer.getLogFilesInitializer(), + new IndexImporterFactoryImpl(), + EmptyMemoryTracker.INSTANCE, + new CursorContextFactory(PageCacheTracer.NULL, EmptyVersionContextSupplier.EMPTY) + ); + } + + @Override + public Input batchInputFrom(CompatInput compatInput) { + return new InputFromCompatInput(compatInput); + } + + @Override + public InputEntityIdVisitor.Long inputEntityLongIdVisitor(IdType idType, ReadableGroups groups) { + switch (idType) { + case ACTUAL -> { + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id); + } + }; + } + case INTEGER -> { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.Long() { + @Override + public void visitNodeId(InputEntityVisitor visitor, long id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, long id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, long id) { + visitor.endId(id, globalGroup); + } + }; + } + default -> throw new IllegalStateException("Unexpected value: " + idType); + } + } + + @Override + public InputEntityIdVisitor.String inputEntityStringIdVisitor(ReadableGroups groups) { + var globalGroup = groups.get(null); + + return new InputEntityIdVisitor.String() { + @Override + public void visitNodeId(InputEntityVisitor visitor, String id) { + visitor.id(id, globalGroup); + } + + @Override + public void visitSourceId(InputEntityVisitor visitor, String id) { + visitor.startId(id, globalGroup); + } + + @Override + public void visitTargetId(InputEntityVisitor visitor, String id) { + visitor.endId(id, globalGroup); + } + }; + } + + @Override + public Setting additionalJvm() { + return BootloaderSettings.additional_jvm; + } + + @Override + public Setting pageCacheMemory() { + return GraphDatabaseSettings.pagecache_memory; + } + + @Override + public Long pageCacheMemoryValue(String value) { + return SettingValueParsers.BYTES.parse(value); + } + + @Override + public ExecutionMonitor invisibleExecutionMonitor() { + return ExecutionMonitor.INVISIBLE; + } + + @Override + public ProcedureSignature procedureSignature( + QualifiedName name, + List inputSignature, + List outputSignature, + Mode mode, + boolean admin, + String deprecated, + String description, + String warning, + boolean eager, + boolean caseInsensitive, + boolean systemProcedure, + boolean internal, + boolean allowExpiredCredentials + ) { + return new ProcedureSignature( + name, + inputSignature, + outputSignature, + mode, + admin, + deprecated, + description, + warning, + eager, + caseInsensitive, + systemProcedure, + internal, + allowExpiredCredentials + ); + } + + @Override + public long getHighestPossibleNodeCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.NODE).orElseGet(read::nodesGetCount); + } + + @Override + public long getHighestPossibleRelationshipCount( + Read read, IdGeneratorFactory idGeneratorFactory + ) { + return countByIdGenerator(idGeneratorFactory, RecordIdType.RELATIONSHIP).orElseGet(read::relationshipsGetCount); + } + + @Override + public String versionLongToString(long storeVersion) { + // copied from org.neo4j.kernel.impl.store.LegacyMetadataHandler.versionLongToString which is private + if (storeVersion == -1) { + return "Unknown"; + } + Bits bits = Bits.bitsFromLongs(new long[]{storeVersion}); + int length = bits.getShort(8); + if (length == 0 || length > 7) { + throw new IllegalArgumentException(format(Locale.ENGLISH, "The read version string length %d is not proper.", length)); + } + char[] result = new char[length]; + for (int i = 0; i < length; i++) { + result[i] = (char) bits.getShort(8); + } + return new String(result); + } + + private static final class InputFromCompatInput implements Input { + private final CompatInput delegate; + + private InputFromCompatInput(CompatInput delegate) { + this.delegate = delegate; + } + + @Override + public InputIterable nodes(Collector badCollector) { + return delegate.nodes(badCollector); + } + + @Override + public InputIterable relationships(Collector badCollector) { + return delegate.relationships(badCollector); + } + + @Override + public IdType idType() { + return delegate.idType(); + } + + @Override + public ReadableGroups groups() { + return delegate.groups(); + } + + @Override + public Estimates calculateEstimates(PropertySizeCalculator propertySizeCalculator) throws IOException { + return delegate.calculateEstimates((values, kernelTransaction) -> propertySizeCalculator.calculateSize( + values, + kernelTransaction.cursorContext(), + kernelTransaction.memoryTracker() + )); + } + } + + @Override + public TestLog testLog() { + return new TestLogImpl(); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getUserLog(LogService logService, Class loggingClass) { + return logService.getUserLog(loggingClass); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Log getInternalLog(LogService logService, Class loggingClass) { + return logService.getInternalLog(loggingClass); + } + + @Override + public Relationship virtualRelationship(long id, Node startNode, Node endNode, RelationshipType type) { + return new VirtualRelationshipImpl(id, startNode, endNode, type); + } + + @Override + public GdsDatabaseManagementServiceBuilder databaseManagementServiceBuilder(Path storeDir) { + return new GdsDatabaseManagementServiceBuilderImpl(storeDir); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats selectRecordFormatForStore( + DatabaseLayout databaseLayout, + FileSystemAbstraction fs, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + return RecordFormatSelector.selectForStore( + (RecordDatabaseLayout) databaseLayout, + fs, + pageCache, + logService.getInternalLogProvider(), + new CursorContextFactory(pageCacheTracer, EMPTY) + ); + } + + @Override + public boolean isNotNumericIndex(IndexCapability indexCapability) { + return !indexCapability.areValueCategoriesAccepted(ValueCategory.NUMBER); + } + + @Override + public void setAllowUpgrades(Config.Builder configBuilder, boolean value) { + } + + @Override + public String defaultRecordFormatSetting() { + return GraphDatabaseSettings.db_format.defaultValue(); + } + + @Override + public void configureRecordFormat(Config.Builder configBuilder, String recordFormat) { + var databaseRecordFormat = recordFormat.toLowerCase(Locale.ENGLISH); + configBuilder.set(GraphDatabaseSettings.db_format, databaseRecordFormat); + } + + @Override + public GdsDatabaseLayout databaseLayout(Config config, String databaseName) { + var storageEngineFactory = StorageEngineFactory.selectStorageEngine(config); + var dbLayout = neo4jLayout(config).databaseLayout(databaseName); + var databaseLayout = storageEngineFactory.formatSpecificDatabaseLayout(dbLayout); + return new GdsDatabaseLayoutImpl(databaseLayout); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public Neo4jLayout neo4jLayout(Config config) { + return Neo4jLayout.of(config); + } + + @Override + public BoltTransactionRunner boltTransactionRunner() { + return new BoltTransactionRunnerImpl(); + } + + @Override + public HostnamePort getLocalBoltAddress(ConnectorPortRegister connectorPortRegister) { + return connectorPortRegister.getLocalAddress(ConnectorType.BOLT); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public SslPolicyLoader createSllPolicyLoader( + FileSystemAbstraction fileSystem, + Config config, + LogService logService + ) { + return SslPolicyLoader.create(fileSystem, config, logService.getInternalLogProvider()); + } + + @Override + @SuppressForbidden(reason = "This is the compat specific use") + public RecordFormats recordFormatSelector( + String databaseName, + Config databaseConfig, + FileSystemAbstraction fs, + LogService logService, + GraphDatabaseService databaseService + ) { + var neo4jLayout = Neo4jLayout.of(databaseConfig); + var recordDatabaseLayout = RecordDatabaseLayout.of(neo4jLayout, databaseName); + return RecordFormatSelector.selectForStoreOrConfigForNewDbs( + databaseConfig, + recordDatabaseLayout, + fs, + GraphDatabaseApiProxy.resolveDependency(databaseService, PageCache.class), + logService.getInternalLogProvider(), + GraphDatabaseApiProxy.resolveDependency(databaseService, CursorContextFactory.class) + ); + } + + @Override + public NamedDatabaseId randomDatabaseId() { + return new TestDatabaseIdRepository().getByName(UUID.randomUUID().toString()).get(); + } + + @Override + public ExecutionMonitor executionMonitor(CompatExecutionMonitor compatExecutionMonitor) { + return new ExecutionMonitor.Adapter( + compatExecutionMonitor.checkIntervalMillis(), + TimeUnit.MILLISECONDS + ) { + + @Override + public void initialize(DependencyResolver dependencyResolver) { + compatExecutionMonitor.initialize(dependencyResolver); + } + + @Override + public void start(StageExecution execution) { + compatExecutionMonitor.start(execution); + } + + @Override + public void end(StageExecution execution, long totalTimeMillis) { + compatExecutionMonitor.end(execution, totalTimeMillis); + } + + @Override + public void done(boolean successful, long totalTimeMillis, String additionalInformation) { + compatExecutionMonitor.done(successful, totalTimeMillis, additionalInformation); + } + + @Override + public void check(StageExecution execution) { + compatExecutionMonitor.check(execution); + } + }; + } + + @Override + @SuppressFBWarnings("NP_LOAD_OF_KNOWN_NULL_VALUE") // We assign nulls because it makes the code more readable + public UserFunctionSignature userFunctionSignature( + QualifiedName name, + List inputSignature, + Neo4jTypes.AnyType type, + String description, + boolean internal, + boolean threadSafe + ) { + String deprecated = null; // no depracation + String category = null; // No predefined categpry (like temporal or math) + var caseInsensitive = false; // case sensitive name match + var isBuiltIn = false; // is built in; never true for GDS + + return new UserFunctionSignature( + name, + inputSignature, + type, + deprecated, + description, + category, + caseInsensitive, + isBuiltIn, + internal, + threadSafe + ); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableProcedure callableProcedure(CompatCallableProcedure procedure) { + return new CallableProcedureImpl(procedure); + } + + @Override + @SuppressForbidden(reason = "This is the compat API") + public CallableUserAggregationFunction callableUserAggregationFunction(CompatUserAggregationFunction function) { + return new CallableUserAggregationFunctionImpl(function); + } + + @Override + public long transactionId(KernelTransactionHandle kernelTransactionHandle) { + return kernelTransactionHandle.getTransactionSequenceNumber(); + } + + @Override + public void reserveNeo4jIds(IdGeneratorFactory generatorFactory, int size, CursorContext cursorContext) { + IdGenerator idGenerator = generatorFactory.get(RecordIdType.NODE); + + idGenerator.nextConsecutiveIdRange(size, false, cursorContext); + } + + @Override + public TransactionalContext newQueryContext( + TransactionalContextFactory contextFactory, + InternalTransaction tx, + String queryText, + MapValue queryParameters + ) { + return contextFactory.newContext(tx, queryText, queryParameters, QueryExecutionConfiguration.DEFAULT_CONFIG); + } + + @Override + public boolean isCompositeDatabase(GraphDatabaseService databaseService) { + var databaseManager = GraphDatabaseApiProxy.resolveDependency(databaseService, FabricDatabaseManager.class); + return databaseManager.isFabricDatabase(GraphDatabaseApiProxy.databaseId(databaseService)); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java new file mode 100644 index 00000000000..849d9a4d5cd --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/NodeLabelIndexLookupImpl.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.common.EntityType; +import org.neo4j.internal.kernel.api.InternalIndexState; +import org.neo4j.internal.kernel.api.SchemaRead; +import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.SchemaDescriptors; +import org.neo4j.kernel.api.KernelTransaction; + +final class NodeLabelIndexLookupImpl { + + static boolean hasNodeLabelIndex(KernelTransaction transaction) { + return NodeLabelIndexLookupImpl.findUsableMatchingIndex( + transaction, + SchemaDescriptors.forAnyEntityTokens(EntityType.NODE) + ) != IndexDescriptor.NO_INDEX; + } + + static IndexDescriptor findUsableMatchingIndex( + KernelTransaction transaction, + SchemaDescriptor schemaDescriptor + ) { + var schemaRead = transaction.schemaRead(); + var iterator = schemaRead.index(schemaDescriptor); + while (iterator.hasNext()) { + var index = iterator.next(); + if (index.getIndexType() == IndexType.LOOKUP && indexIsOnline(schemaRead, index)) { + return index; + } + } + return IndexDescriptor.NO_INDEX; + } + + private static boolean indexIsOnline(SchemaRead schemaRead, IndexDescriptor index) { + var state = InternalIndexState.FAILED; + try { + state = schemaRead.indexGetState(index); + } catch (IndexNotFoundKernelException e) { + // Well the index should always exist here, but if we didn't find it while checking the state, + // then we obviously don't want to use it. + } + return state == InternalIndexState.ONLINE; + } + + private NodeLabelIndexLookupImpl() {} +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java new file mode 100644 index 00000000000..2a8da868aeb --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/PartitionedStoreScan.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.NodeLabelIndexCursor; +import org.neo4j.internal.kernel.api.PartitionedScan; +import org.neo4j.kernel.api.KernelTransaction; + +final class PartitionedStoreScan implements StoreScan { + private final PartitionedScan scan; + + PartitionedStoreScan(PartitionedScan scan) { + this.scan = scan; + } + + static int getNumberOfPartitions(long nodeCount, int batchSize) { + int numberOfPartitions; + if (nodeCount > 0) { + // ceil div to try to get enough partitions so a single one does + // not include more nodes than batchSize + long partitions = ((nodeCount - 1) / batchSize) + 1; + + // value must be positive + if (partitions < 1) { + partitions = 1; + } + + numberOfPartitions = (int) Long.min(Integer.MAX_VALUE, partitions); + } else { + // we have no partitions to scan, but the value must still be positive + numberOfPartitions = 1; + } + return numberOfPartitions; + } + + @Override + public boolean reserveBatch(NodeLabelIndexCursor cursor, KernelTransaction ktx) { + return scan.reservePartition(cursor, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java new file mode 100644 index 00000000000..a24847fc2de --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ReferencePropertyReference.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.PropertyReference; +import org.neo4j.storageengine.api.Reference; + +import java.util.Objects; + +public final class ReferencePropertyReference implements PropertyReference { + + private static final PropertyReference EMPTY = new ReferencePropertyReference(null); + + public final Reference reference; + + private ReferencePropertyReference(Reference reference) { + this.reference = reference; + } + + public static PropertyReference of(Reference reference) { + return new ReferencePropertyReference(Objects.requireNonNull(reference)); + } + + public static PropertyReference empty() { + return EMPTY; + } + + @Override + public boolean isEmpty() { + return reference == null; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java new file mode 100644 index 00000000000..c108e8bec61 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/ScanBasedStoreScanImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.StoreScan; +import org.neo4j.internal.kernel.api.Cursor; +import org.neo4j.internal.kernel.api.Scan; +import org.neo4j.kernel.api.KernelTransaction; + +public final class ScanBasedStoreScanImpl implements StoreScan { + private final Scan scan; + private final int batchSize; + + public ScanBasedStoreScanImpl(Scan scan, int batchSize) { + this.scan = scan; + this.batchSize = batchSize; + } + + @Override + public boolean reserveBatch(C cursor, KernelTransaction ktx) { + return scan.reserveBatch(cursor, batchSize, ktx.cursorContext(), ktx.securityContext().mode()); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java new file mode 100644 index 00000000000..c39ad8aa279 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.gds.compat.SettingProxyFactory; + +@ServiceProvider +public final class SettingProxyFactoryImpl implements SettingProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public SettingProxyApi load() { + return new SettingProxyImpl(); + } + + @Override + public String description() { + return "Neo4j Settings 5.9"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java new file mode 100644 index 00000000000..662daf64e73 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/SettingProxyImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.configuration.SettingBuilder; +import org.neo4j.dbms.systemgraph.TopologyGraphDbmsModel; +import org.neo4j.gds.compat.DatabaseMode; +import org.neo4j.gds.compat.SettingProxyApi; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.config.Setting; +import org.neo4j.kernel.impl.factory.GraphDatabaseFacade; +import org.neo4j.kernel.internal.GraphDatabaseAPI; + +public class SettingProxyImpl implements SettingProxyApi { + + @Override + public Setting setting(org.neo4j.gds.compat.Setting setting) { + var builder = SettingBuilder.newBuilder(setting.name(), setting.parser(), setting.defaultValue()); + if (setting.dynamic()) { + builder = builder.dynamic(); + } + if (setting.immutable()) { + builder = builder.immutable(); + } + setting.dependency().ifPresent(builder::setDependency); + setting.constraints().forEach(builder::addConstraint); + return builder.build(); + } + + @Override + public DatabaseMode databaseMode(Config config, GraphDatabaseService databaseService) { + return switch (((GraphDatabaseAPI) databaseService).mode()) { + case RAFT -> DatabaseMode.CORE; + case REPLICA -> DatabaseMode.READ_REPLICA; + case SINGLE -> DatabaseMode.SINGLE; + case VIRTUAL -> throw new UnsupportedOperationException("What's a virtual database anyway?"); + }; + } + + @Override + public void setDatabaseMode(Config config, DatabaseMode databaseMode, GraphDatabaseService databaseService) { + // super hacky, there is no way to set the mode of a database without restarting it + if (!(databaseService instanceof GraphDatabaseFacade db)) { + throw new IllegalArgumentException( + "Cannot set database mode on a database that is not a GraphDatabaseFacade"); + } + try { + var modeField = GraphDatabaseFacade.class.getDeclaredField("mode"); + modeField.setAccessible(true); + modeField.set(db, switch (databaseMode) { + case CORE -> TopologyGraphDbmsModel.HostedOnMode.RAFT; + case READ_REPLICA -> TopologyGraphDbmsModel.HostedOnMode.REPLICA; + case SINGLE -> TopologyGraphDbmsModel.HostedOnMode.SINGLE; + }); + } catch (NoSuchFieldException e) { + throw new RuntimeException( + "Could not set the mode field because it no longer exists. This compat layer needs to be updated.", + e + ); + } catch (IllegalAccessException e) { + throw new RuntimeException("Could not get the permissions to set the mode field.", e); + } + } + + @Override + public String secondaryModeName() { + return "Secondary"; + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java new file mode 100644 index 00000000000..7d09c9ce756 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/TestLogImpl.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.TestLog; + +import java.util.ArrayList; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; + +public class TestLogImpl implements TestLog { + + private final ConcurrentMap> messages; + + TestLogImpl() { + messages = new ConcurrentHashMap<>(3); + } + + @Override + public void assertContainsMessage(String level, String fragment) { + if (!containsMessage(level, fragment)) { + throw new RuntimeException( + String.format( + Locale.US, + "Expected log output to contain `%s` for log level `%s`%nLog messages:%n%s", + fragment, + level, + String.join("\n", messages.get(level)) + ) + ); + } + } + + @Override + public boolean containsMessage(String level, String fragment) { + ConcurrentLinkedQueue messageList = messages.getOrDefault(level, new ConcurrentLinkedQueue<>()); + return messageList.stream().anyMatch((message) -> message.contains(fragment)); + } + + @Override + public boolean hasMessages(String level) { + return !messages.getOrDefault(level, new ConcurrentLinkedQueue<>()).isEmpty(); + } + + @Override + public ArrayList getMessages(String level) { + return new ArrayList<>(messages.getOrDefault(level, new ConcurrentLinkedQueue<>())); + } + + @SuppressForbidden(reason = "test log can print") + public void printMessages() { + System.out.println("TestLog Messages: " + messages); + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public void debug(String message) { + logMessage(DEBUG, message); + } + + @Override + public void debug(String message, Throwable throwable) { + debug(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void debug(String format, Object... arguments) { + debug(String.format(Locale.US, format, arguments)); + } + + @Override + public void info(String message) { + logMessage(INFO, message); + } + + @Override + public void info(String message, Throwable throwable) { + info(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void info(String format, Object... arguments) { + info(String.format(Locale.US, format, arguments)); + } + + @Override + public void warn(String message) { + logMessage(WARN, message); + } + + @Override + public void warn(String message, Throwable throwable) { + warn(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void warn(String format, Object... arguments) { + warn(String.format(Locale.US, format, arguments)); + } + + @Override + public void error(String message) { + logMessage(ERROR, message); + } + + @Override + public void error(String message, Throwable throwable) { + error(String.format(Locale.US, "%s - %s", message, throwable.getMessage())); + } + + @Override + public void error(String format, Object... arguments) { + error(String.format(Locale.US, format, arguments)); + } + + private void logMessage(String level, String message) { + messages.computeIfAbsent( + level, + (ignore) -> new ConcurrentLinkedQueue<>() + ).add(message); + } +} diff --git a/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java new file mode 100644 index 00000000000..07708b93b25 --- /dev/null +++ b/compatibility/5.9/neo4j-kernel-adapter/src/main/java17/org/neo4j/gds/compat/_59/VirtualRelationshipImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.VirtualRelationship; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.RelationshipType; + +public class VirtualRelationshipImpl extends VirtualRelationship { + + VirtualRelationshipImpl( + long id, + Node startNode, + Node endNode, + RelationshipType type + ) { + super(id, startNode, endNode, type); + } + + @Override + public String getElementId() { + return Long.toString(getId()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/build.gradle b/compatibility/5.9/storage-engine-adapter/build.gradle new file mode 100644 index 00000000000..17bd19ef579 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'java-library' +apply plugin: 'me.champeau.mrjar' + +description = 'Neo4j Graph Data Science :: Storage Engine Adapter 5.9' + +group = 'org.neo4j.gds' + +// for all 5.x versions +if (ver.'neo4j'.startsWith('5.')) { + sourceSets { + main { + java { + srcDirs = ['src/main/java17'] + } + } + } + + dependencies { + annotationProcessor project(':annotations') + annotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + annotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + compileOnly project(':annotations') + compileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + compileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + compileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + + implementation project(':core') + implementation project(':storage-engine-adapter-api') + implementation project(':config-api') + implementation project(':string-formatting') + } +} else { + multiRelease { + targetVersions 11, 17 + } + + if (!project.hasProperty('no-forbidden-apis')) { + forbiddenApisJava17 { + exclude('**') + } + } + + dependencies { + annotationProcessor group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'annotations', version: ver.'neo4j' + compileOnly group: 'org.neo4j', name: 'neo4j-kernel-api', version: ver.'neo4j' + + implementation project(':neo4j-adapter') + implementation project(':storage-engine-adapter-api') + + java17AnnotationProcessor project(':annotations') + java17AnnotationProcessor group: 'org.immutables', name: 'value', version: ver.'immutables' + java17AnnotationProcessor group: 'org.neo4j', name: 'annotations', version: neos.'5.9' + + java17CompileOnly project(':annotations') + java17CompileOnly group: 'org.immutables', name: 'value-annotations', version: ver.'immutables' + java17CompileOnly group: 'org.neo4j', name: 'neo4j', version: neos.'5.9' + java17CompileOnly group: 'org.neo4j', name: 'neo4j-record-storage-engine', version: neos.'5.9' + + java17Implementation project(':core') + java17Implementation project(':storage-engine-adapter-api') + java17Implementation project(':config-api') + java17Implementation project(':string-formatting') + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..4bdc7bd36f8 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java @@ -0,0 +1,268 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.id.IdController; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.lock.LockService; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogVersionRepository; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.TransactionIdStore; +import org.neo4j.storageengine.migration.RollingUpgradeCompatibility; +import org.neo4j.storageengine.migration.SchemaRuleMigrationAccess; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.token.TokenHolders; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + @Override + public String name() { + return "unsupported59"; + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + PageCacheTracer pageCacheTracer + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(String storeVersion) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreVersion versionInformation(StoreId storeId) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public RollingUpgradeCompatibility rollingUpgradeCompatibility() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fs, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + PageCacheTracer cacheTracer, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + IdController idController, + DatabaseHealth databaseHealth, + LogProvider internalLogProvider, + LogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + PageCacheTracer cacheTracer, + boolean createStoreIfNotExists, + DatabaseReadOnlyChecker readOnlyChecker, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List listStorageFiles(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) throws + IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout, PageCache pageCache) { + return false; + } + + @Override + public TransactionIdStore readOnlyTransactionIdStore( + FileSystemAbstraction filySystem, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public LogVersionRepository readOnlyLogVersionRepository( + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer cacheTracer, + DatabaseReadOnlyChecker readOnlyChecker + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StoreId storeId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public void setStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + StoreId storeId, + long upgradeTxChecksum, + long upgradeTxCommitTimestamp + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public void setExternalStoreUUID( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext, + UUID externalStoreId + ) throws IOException { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public SchemaRuleMigrationAccess schemaRuleMigrationAccess( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + LogService logService, + String recordFormats, + PageCacheTracer cacheTracer, + CursorContext cursorContext, + MemoryTracker memoryTracker + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + Config config, + DatabaseLayout databaseLayout, + CursorContext cursorContext + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache + ) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public CommandReaderFactory commandReaderFactory() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public DatabaseLayout databaseLayout(Neo4jLayout neo4jLayout, String databaseName) { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..e43773a19d3 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return false; + } + + @Override + public StorageEngineProxyApi load() { + throw new UnsupportedOperationException("5.9 storage engine requires JDK17"); + } + + @Override + public String description() { + return "Storage Engine 5.9"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java new file mode 100644 index 00000000000..6bad2296b00 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCommandCreationContextImpl.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.KernelVersionProvider; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.cursor.StoreCursors; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class InMemoryCommandCreationContextImpl implements CommandCreationContext { + + private final AtomicLong schemaTokens; + private final AtomicInteger propertyTokens; + private final AtomicInteger labelTokens; + private final AtomicInteger typeTokens; + + InMemoryCommandCreationContextImpl() { + this.schemaTokens = new AtomicLong(0); + this.propertyTokens = new AtomicInteger(0); + this.labelTokens = new AtomicInteger(0); + this.typeTokens = new AtomicInteger(0); + } + + @Override + public long reserveNode() { + throw new UnsupportedOperationException("Creating nodes is not supported"); + } + + @Override + public long reserveRelationship( + long sourceNode, + long targetNode, + int relationshipType, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + throw new UnsupportedOperationException("Creating relationships is not supported"); + } + + @Override + public long reserveSchema() { + return schemaTokens.getAndIncrement(); + } + + @Override + public int reserveLabelTokenId() { + return labelTokens.getAndIncrement(); + } + + @Override + public int reservePropertyKeyTokenId() { + return propertyTokens.getAndIncrement(); + } + + @Override + public int reserveRelationshipTypeTokenId() { + return typeTokens.getAndIncrement(); + } + + @Override + public void close() { + + } + + @Override + public void initialize( + KernelVersionProvider kernelVersionProvider, + CursorContext cursorContext, + StoreCursors storeCursors, + Supplier oldestActiveTransactionSequenceNumber, + ResourceLocker locks, + Supplier lockTracer + ) { + + } + + @Override + public KernelVersion kernelVersion() { + // NOTE: Double-check if this is still correct when you copy this into a new compat layer + return KernelVersion.getLatestVersion(Config.newBuilder().build()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java new file mode 100644 index 00000000000..7cf4f836b7d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryCountsStoreImpl.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.documented.ReporterFactory; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.counts.CountsStorage; +import org.neo4j.counts.CountsVisitor; +import org.neo4j.gds.NodeLabel; +import org.neo4j.gds.api.GraphStore; +import org.neo4j.internal.helpers.progress.ProgressMonitorFactory; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.FileFlushEvent; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.TokenNotFoundException; + +public class InMemoryCountsStoreImpl implements CountsStorage, CountsAccessor { + + private final GraphStore graphStore; + private final TokenHolders tokenHolders; + + public InMemoryCountsStoreImpl( + GraphStore graphStore, + TokenHolders tokenHolders + ) { + + this.graphStore = graphStore; + this.tokenHolders = tokenHolders; + } + + @Override + public void start( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + + } + + @Override + public void checkpoint(FileFlushEvent fileFlushEvent, CursorContext cursorContext) { + + } + + @Override + public long nodeCount(int labelId, CursorContext cursorContext) { + if (labelId == -1) { + return graphStore.nodeCount(); + } + + String nodeLabel; + try { + nodeLabel = tokenHolders.labelTokens().getTokenById(labelId).name(); + } catch (TokenNotFoundException e) { + throw new RuntimeException(e); + } + return graphStore.nodes().nodeCount(NodeLabel.of(nodeLabel)); + } + + @Override + public long relationshipCount(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + // TODO: this is quite wrong + return graphStore.relationshipCount(); + } + + @Override + public boolean consistencyCheck( + ReporterFactory reporterFactory, + CursorContextFactory contextFactory, + int numThreads, + ProgressMonitorFactory progressMonitorFactory + ) { + return true; + } + + @Override + public CountsAccessor.Updater apply(long txId, boolean isLast, CursorContext cursorContext) { + throw new UnsupportedOperationException("Updates are not supported"); + } + + @Override + public void close() { + + } + + @Override + public void accept(CountsVisitor visitor, CursorContext cursorContext) { + tokenHolders.labelTokens().getAllTokens().forEach(labelToken -> { + visitor.visitNodeCount(labelToken.id(), nodeCount(labelToken.id(), cursorContext)); + }); + + visitor.visitRelationshipCount(-1, -1, -1, graphStore.relationshipCount()); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java new file mode 100644 index 00000000000..77f9b72465f --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryMetaDataProviderImpl.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.internal.recordstorage.InMemoryLogVersionRepository59; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.ExternalStoreId; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionId; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +public class InMemoryMetaDataProviderImpl implements MetadataProvider { + + private final ExternalStoreId externalStoreId; + private final InMemoryLogVersionRepository59 logVersionRepository; + private final InMemoryTransactionIdStoreImpl transactionIdStore; + + InMemoryMetaDataProviderImpl() { + this.logVersionRepository = new InMemoryLogVersionRepository59(); + this.externalStoreId = new ExternalStoreId(UUID.randomUUID()); + this.transactionIdStore = new InMemoryTransactionIdStoreImpl(); + } + + @Override + public ExternalStoreId getExternalStoreId() { + return this.externalStoreId; + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + return this.transactionIdStore.getLastClosedTransaction(); + } + + @Override + public void setCurrentLogVersion(long version) { + logVersionRepository.setCurrentLogVersion(version); + } + + @Override + public long incrementAndGetVersion() { + return logVersionRepository.incrementAndGetVersion(); + } + + @Override + public void setCheckpointLogVersion(long version) { + logVersionRepository.setCheckpointLogVersion(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return logVersionRepository.incrementAndGetCheckpointLogVersion(); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + transactionIdStore.transactionCommitted(transactionId, checksum, commitTimestamp, consensusIndex); + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + transactionIdStore.setLastCommittedAndClosedTransactionId( + transactionId, + checksum, + commitTimestamp, + consensusIndex, + byteOffset, + logVersion + ); + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.transactionClosed( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.transactionIdStore.resetLastClosedTransaction( + transactionId, + logVersion, + byteOffset, + checksum, + commitTimestamp, + consensusIndex + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + public void regenerateMetadata(StoreId storeId, UUID externalStoreUUID, CursorContext cursorContext) { + } + + @Override + public StoreId getStoreId() { + return StoreId.UNKNOWN; + } + + @Override + public void close() throws IOException { + } + + @Override + public long getCurrentLogVersion() { + return this.logVersionRepository.getCurrentLogVersion(); + } + + @Override + public long getCheckpointLogVersion() { + return this.logVersionRepository.getCheckpointLogVersion(); + } + + @Override + public long nextCommittingTransactionId() { + return this.transactionIdStore.nextCommittingTransactionId(); + } + + @Override + public long committingTransactionId() { + return this.transactionIdStore.committingTransactionId(); + } + + @Override + public long getLastCommittedTransactionId() { + return this.transactionIdStore.getLastCommittedTransactionId(); + } + + @Override + public TransactionId getLastCommittedTransaction() { + return this.transactionIdStore.getLastCommittedTransaction(); + } + + @Override + public long getLastClosedTransactionId() { + return this.transactionIdStore.getLastClosedTransactionId(); + } + + @Override + public Optional getDatabaseIdUuid(CursorContext cursorTracer) { + throw new IllegalStateException("Not supported"); + } + + @Override + public void setDatabaseIdUuid(UUID uuid, CursorContext cursorContext) { + throw new IllegalStateException("Not supported"); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java new file mode 100644 index 00000000000..15d394b1586 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodeCursor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.api.GraphStore; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.Degrees; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodeCursor extends AbstractInMemoryNodeCursor { + + public InMemoryNodeCursor(GraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public boolean hasLabel() { + return hasAtLeastOneLabelForCurrentNode(); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor, PropertySelection selection) { + propertyCursor.initNodeProperties(propertiesReference(), selection); + } + + @Override + public void properties(StoragePropertyCursor propertyCursor) { + properties(propertyCursor, PropertySelection.ALL_PROPERTIES); + } + + @Override + public boolean supportsFastRelationshipsTo() { + return false; + } + + @Override + public void relationshipsTo( + StorageRelationshipTraversalCursor storageRelationshipTraversalCursor, + RelationshipSelection relationshipSelection, + long neighbourNodeReference + ) { + throw new UnsupportedOperationException(); + } + + @Override + public void degrees(RelationshipSelection selection, Degrees.Mutator mutator) { + } + + @Override + public boolean scanBatch(AllNodeScan allNodeScan, long sizeHint) { + return super.scanBatch(allNodeScan, (int) sizeHint); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java new file mode 100644 index 00000000000..43b8fd4837a --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryNodePropertyCursor.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.token.TokenHolders; + +public class InMemoryNodePropertyCursor extends AbstractInMemoryNodePropertyCursor { + + public InMemoryNodePropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + reset(); + setId(((LongReference) reference).id); + setPropertySelection(new InMemoryPropertySelectionImpl(selection)); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java new file mode 100644 index 00000000000..2b6499a1f96 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertyCursor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryPropertyCursor extends AbstractInMemoryPropertyCursor { + + public InMemoryPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(reference, selection); + } + + @Override + public void initNodeProperties(StorageNodeCursor nodeCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryNodePropertyCursor)) { + this.delegate = new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryNodePropertyCursor) delegate).initNodeProperties(nodeCursor, selection); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(relationshipCursor, selection); + } + + @Override + public void initRelationshipProperties(Reference reference, PropertySelection selection, long ownerReference) { + if (this.delegate == null || !(this.delegate instanceof InMemoryRelationshipPropertyCursor)) { + this.delegate = new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + ((InMemoryRelationshipPropertyCursor) delegate).initRelationshipProperties(reference, selection); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java new file mode 100644 index 00000000000..5724e2829d3 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryPropertySelectionImpl.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.InMemoryPropertySelection; +import org.neo4j.storageengine.api.PropertySelection; + +public class InMemoryPropertySelectionImpl implements InMemoryPropertySelection { + + private final PropertySelection propertySelection; + + public InMemoryPropertySelectionImpl(PropertySelection propertySelection) {this.propertySelection = propertySelection;} + + @Override + public boolean isLimited() { + return propertySelection.isLimited(); + } + + @Override + public int numberOfKeys() { + return propertySelection.numberOfKeys(); + } + + @Override + public int key(int index) { + return propertySelection.key(index); + } + + @Override + public boolean test(int key) { + return propertySelection.test(key); + } + + @Override + public boolean isKeysOnly() { + return propertySelection.isKeysOnly(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java new file mode 100644 index 00000000000..806f0b25659 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipPropertyCursor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.storageengine.InMemoryRelationshipCursor; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StorageRelationshipCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipPropertyCursor extends AbstractInMemoryRelationshipPropertyCursor { + + InMemoryRelationshipPropertyCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public void initNodeProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + + } + + @Override + public void initRelationshipProperties( + Reference reference, PropertySelection propertySelection, long ownerReference + ) { + var relationshipId = ((LongReference) reference).id; + var relationshipCursor = new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + relationshipCursor.single(relationshipId); + relationshipCursor.next(); + relationshipCursor.properties(this, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public void initRelationshipProperties(StorageRelationshipCursor relationshipCursor, PropertySelection selection) { + var inMemoryRelationshipCursor = (InMemoryRelationshipCursor) relationshipCursor; + inMemoryRelationshipCursor.properties(this, selection); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java new file mode 100644 index 00000000000..6f0c089ee8d --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipScanCursor.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipScanCursor extends AbstractInMemoryRelationshipScanCursor { + + public InMemoryRelationshipScanCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + super(graphStore, tokenHolders); + } + + @Override + public void single(long reference, long sourceNodeReference, int type, long targetNodeReference) { + single(reference); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor storagePropertyCursor, PropertySelection propertySelection + ) { + properties(storagePropertyCursor, new InMemoryPropertySelectionImpl(propertySelection)); + } + + @Override + public boolean scanBatch(AllRelationshipsScan allRelationshipsScan, long sizeHint) { + return super.scanBatch(allRelationshipsScan, (int) sizeHint); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java new file mode 100644 index 00000000000..6cd7ebf0310 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryRelationshipTraversalCursor.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.storageengine.api.LongReference; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.Reference; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.token.TokenHolders; + +public class InMemoryRelationshipTraversalCursor extends AbstractInMemoryRelationshipTraversalCursor { + + public InMemoryRelationshipTraversalCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + super(graphStore, tokenHolders); + } + + @Override + public Reference propertiesReference() { + return LongReference.longReference(getId()); + } + + @Override + public void properties( + StoragePropertyCursor propertyCursor, PropertySelection selection + ) { + properties(propertyCursor, new InMemoryPropertySelectionImpl(selection)); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java new file mode 100644 index 00000000000..ad9133ba7ea --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineFactory.java @@ -0,0 +1,558 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.set.ImmutableSet; +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.configuration.Config; +import org.neo4j.consistency.checking.ConsistencyFlags; +import org.neo4j.consistency.report.ConsistencySummaryStatistics; +import org.neo4j.dbms.database.readonly.DatabaseReadOnlyChecker; +import org.neo4j.function.ThrowingSupplier; +import org.neo4j.gds.annotation.SuppressForbidden; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; +import org.neo4j.internal.batchimport.AdditionalInitialIds; +import org.neo4j.internal.batchimport.BatchImporter; +import org.neo4j.internal.batchimport.Configuration; +import org.neo4j.internal.batchimport.IncrementalBatchImporter; +import org.neo4j.internal.batchimport.IndexImporterFactory; +import org.neo4j.internal.batchimport.Monitor; +import org.neo4j.internal.batchimport.ReadBehaviour; +import org.neo4j.internal.batchimport.input.Collector; +import org.neo4j.internal.batchimport.input.Input; +import org.neo4j.internal.batchimport.input.LenientStoreInput; +import org.neo4j.internal.id.IdGeneratorFactory; +import org.neo4j.internal.id.ScanOnOpenReadOnlyIdGeneratorFactory; +import org.neo4j.internal.recordstorage.InMemoryStorageCommandReaderFactory59; +import org.neo4j.internal.recordstorage.StoreTokens; +import org.neo4j.internal.schema.IndexConfigCompleter; +import org.neo4j.internal.schema.SchemaRule; +import org.neo4j.internal.schema.SchemaState; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.layout.Neo4jLayout; +import org.neo4j.io.layout.recordstorage.RecordDatabaseLayout; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.context.CursorContextFactory; +import org.neo4j.io.pagecache.tracing.PageCacheTracer; +import org.neo4j.kernel.KernelVersionRepository; +import org.neo4j.kernel.api.index.IndexProvidersAccess; +import org.neo4j.kernel.impl.api.index.IndexProviderMap; +import org.neo4j.kernel.impl.locking.LockManager; +import org.neo4j.kernel.impl.store.MetaDataStore; +import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.StoreFactory; +import org.neo4j.kernel.impl.store.StoreType; +import org.neo4j.kernel.impl.store.cursor.CachedStoreCursors; +import org.neo4j.kernel.impl.transaction.log.LogTailLogVersionsMetadata; +import org.neo4j.kernel.impl.transaction.log.LogTailMetadata; +import org.neo4j.lock.LockService; +import org.neo4j.logging.InternalLog; +import org.neo4j.logging.InternalLogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.logging.internal.LogService; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.monitoring.DatabaseHealth; +import org.neo4j.scheduler.JobScheduler; +import org.neo4j.storageengine.api.CommandReaderFactory; +import org.neo4j.storageengine.api.ConstraintRuleAccessor; +import org.neo4j.storageengine.api.LogFilesInitializer; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.SchemaRule44; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageFilesState; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; +import org.neo4j.storageengine.migration.StoreMigrationParticipant; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.DelegatingTokenHolder; +import org.neo4j.token.ReadOnlyTokenCreator; +import org.neo4j.token.TokenHolders; +import org.neo4j.token.api.NamedToken; +import org.neo4j.token.api.TokenHolder; +import org.neo4j.token.api.TokensLoader; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UncheckedIOException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.time.Clock; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; + +@ServiceProvider +public class InMemoryStorageEngineFactory implements StorageEngineFactory { + + static final String IN_MEMORY_STORAGE_ENGINE_NAME = "in-memory-59"; + + public InMemoryStorageEngineFactory() { + StorageEngineProxyApi.requireNeo4jVersion(Neo4jVersion.V_5_9, StorageEngineFactory.class); + } + + // Record storage = 0, Freki = 1 + // Let's leave some room for future storage engines + // This arbitrary seems quite future-proof + public static final byte ID = 42; + + @Override + public byte id() { + return ID; + } + + @Override + public boolean storageExists(FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout) { + return false; + } + + @Override + public StorageEngine instantiate( + FileSystemAbstraction fs, + Clock clock, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + TokenHolders tokenHolders, + SchemaState schemaState, + ConstraintRuleAccessor constraintSemantics, + IndexConfigCompleter indexConfigCompleter, + LockService lockService, + IdGeneratorFactory idGeneratorFactory, + DatabaseHealth databaseHealth, + InternalLogProvider internalLogProvider, + InternalLogProvider userLogProvider, + RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, + LogTailMetadata logTailMetadata, + KernelVersionRepository kernelVersionRepository, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + idGeneratorFactory, + pageCache, + pageCacheTracer, + fs, + internalLogProvider, + contextFactory, + false, + logTailMetadata + ); + + factory.openNeoStores(StoreType.LABEL_TOKEN).close(); + + return new InMemoryStorageEngineImpl( + databaseLayout, + tokenHolders + ); + } + + @Override + public Optional databaseIdUuid( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache, CursorContext cursorContext + ) { + var fieldAccess = MetaDataStore.getFieldAccess( + pageCache, + RecordDatabaseLayout.convert(databaseLayout).metadataStore(), + databaseLayout.getDatabaseName(), + cursorContext + ); + + try { + return fieldAccess.readDatabaseUUID(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public List migrationParticipants( + FileSystemAbstraction fileSystemAbstraction, + Config config, + PageCache pageCache, + JobScheduler jobScheduler, + LogService logService, + MemoryTracker memoryTracker, + PageCacheTracer pageCacheTracer, + CursorContextFactory cursorContextFactory, + boolean b + ) { + return List.of(); + } + + @Override + public DatabaseLayout databaseLayout( + Neo4jLayout neo4jLayout, String databaseName + ) { + return RecordDatabaseLayout.of(neo4jLayout, databaseName); + } + + @Override + public DatabaseLayout formatSpecificDatabaseLayout(DatabaseLayout plainLayout) { + return databaseLayout(plainLayout.getNeo4jLayout(), plainLayout.getDatabaseName()); + } + + @SuppressForbidden(reason = "This is the compat layer and we don't really need to go through the proxy") + @Override + public BatchImporter batchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCacheTracer pageCacheTracer, + Configuration configuration, + LogService logService, + PrintStream printStream, + boolean b, + AdditionalInitialIds additionalInitialIds, + Config config, + Monitor monitor, + JobScheduler jobScheduler, + Collector collector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory cursorContextFactory + ) { + throw new UnsupportedOperationException("Batch Import into GDS is not supported"); + } + + @Override + public Input asBatchImporterInput( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + MemoryTracker memoryTracker, + ReadBehaviour readBehaviour, + boolean b, + CursorContextFactory cursorContextFactory, + LogTailMetadata logTailMetadata + ) { + NeoStores neoStores = (new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + logTailMetadata + )).openAllNeoStores(); + return new LenientStoreInput( + neoStores, + readBehaviour.decorateTokenHolders(this.loadReadOnlyTokens(neoStores, true, cursorContextFactory)), + true, + cursorContextFactory, + readBehaviour + ); + } + + @Override + public long optimalAvailableConsistencyCheckerMemory( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache + ) { + return 0; + } + + @Override + public String name() { + return IN_MEMORY_STORAGE_ENGINE_NAME; + } + + @Override + public Set supportedFormats(boolean includeFormatsUnderDevelopment) { + return Set.of(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public boolean supportedFormat(String format, boolean includeFormatsUnderDevelopment) { + return format.equals(IN_MEMORY_STORAGE_ENGINE_NAME); + } + + @Override + public MetadataProvider transactionMetaDataStore( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + DatabaseReadOnlyChecker readOnlyChecker, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata, + PageCacheTracer pageCacheTracer + ) { + return new InMemoryMetaDataProviderImpl(); + } + + @Override + public StoreVersionCheck versionCheck( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + LogService logService, + CursorContextFactory cursorContextFactory + ) { + return new InMemoryVersionCheck(); + } + + @Override + public List loadSchemaRules( + FileSystemAbstraction fileSystemAbstraction, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + boolean b, + Function function, + CursorContextFactory cursorContextFactory + ) { + return List.of(); + } + + @Override + public List load44SchemaRules( + FileSystemAbstraction fs, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + Config config, + DatabaseLayout databaseLayout, + CursorContextFactory contextFactory, + LogTailLogVersionsMetadata logTailMetadata + ) { + return List.of(); + } + + @Override + public TokenHolders loadReadOnlyTokens( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + PageCacheTracer pageCacheTracer, + boolean lenient, + CursorContextFactory cursorContextFactory + ) { + StoreFactory factory = new StoreFactory( + databaseLayout, + config, + new ScanOnOpenReadOnlyIdGeneratorFactory(), + pageCache, + pageCacheTracer, + fileSystemAbstraction, + NullLogProvider.getInstance(), + cursorContextFactory, + false, + LogTailMetadata.EMPTY_LOG_TAIL + ); + try ( NeoStores stores = factory.openNeoStores( + StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME, + StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, + StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME ) ) + { + return loadReadOnlyTokens(stores, lenient, cursorContextFactory); + } + } + + private TokenHolders loadReadOnlyTokens( + NeoStores stores, + boolean lenient, + CursorContextFactory cursorContextFactory + ) + { + try ( var cursorContext = cursorContextFactory.create("loadReadOnlyTokens"); + var storeCursors = new CachedStoreCursors( stores, cursorContext ) ) + { + stores.start( cursorContext ); + TokensLoader loader = lenient ? StoreTokens.allReadableTokens( stores ) : StoreTokens.allTokens( stores ); + TokenHolder propertyKeys = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_PROPERTY_KEY ); + TokenHolder labels = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_LABEL ); + TokenHolder relationshipTypes = new DelegatingTokenHolder( ReadOnlyTokenCreator.READ_ONLY, TokenHolder.TYPE_RELATIONSHIP_TYPE ); + + propertyKeys.setInitialTokens( lenient ? unique( loader.getPropertyKeyTokens( storeCursors ) ) : loader.getPropertyKeyTokens( storeCursors ) ); + labels.setInitialTokens( lenient ? unique( loader.getLabelTokens( storeCursors ) ) : loader.getLabelTokens( storeCursors ) ); + relationshipTypes.setInitialTokens( + lenient ? unique( loader.getRelationshipTypeTokens( storeCursors ) ) : loader.getRelationshipTypeTokens( storeCursors ) ); + return new TokenHolders( propertyKeys, labels, relationshipTypes ); + } + catch ( IOException e ) + { + throw new UncheckedIOException( e ); + } + } + + private static List unique( List tokens ) + { + if ( !tokens.isEmpty() ) + { + Set names = new HashSet<>( tokens.size() ); + int i = 0; + while ( i < tokens.size() ) + { + if ( names.add( tokens.get( i ).name() ) ) + { + i++; + } + else + { + // Remove the token at the given index, by replacing it with the last token in the list. + // This changes the order of elements, but can be done in constant time instead of linear time. + int lastIndex = tokens.size() - 1; + NamedToken endToken = tokens.remove( lastIndex ); + if ( i < lastIndex ) + { + tokens.set( i, endToken ); + } + } + } + } + return tokens; + } + + @Override + public CommandReaderFactory commandReaderFactory() { + return InMemoryStorageCommandReaderFactory59.INSTANCE; + } + + @Override + public void consistencyCheck( + FileSystemAbstraction fileSystem, + DatabaseLayout layout, + Config config, + PageCache pageCache, + IndexProviderMap indexProviders, + InternalLog log, + ConsistencySummaryStatistics summary, + int numberOfThreads, + long maxOffHeapCachingMemory, + OutputStream progressOutput, + boolean verbose, + ConsistencyFlags flags, + CursorContextFactory contextFactory, + PageCacheTracer pageCacheTracer, + LogTailMetadata logTailMetadata + ) { + // we can do no-op, since our "database" is _always_ consistent + } + + @Override + public ImmutableSet getStoreOpenOptions( + FileSystemAbstraction fs, + PageCache pageCache, + DatabaseLayout layout, + CursorContextFactory contextFactory + ) { + // Not sure about this, empty set is returned when the store files are in `little-endian` format + // See: `org.neo4j.kernel.impl.store.format.PageCacheOptionsSelector.select` + return Sets.immutable.empty(); + } + + @Override + public StoreId retrieveStoreId( + FileSystemAbstraction fs, + DatabaseLayout databaseLayout, + PageCache pageCache, + CursorContext cursorContext + ) throws IOException { + return StoreId.retrieveFromStore(fs, databaseLayout, pageCache, cursorContext); + } + + + @Override + public Optional versionInformation(StoreVersionIdentifier storeVersionIdentifier) { + return Optional.of(new InMemoryStoreVersion()); + } + + @Override + public void resetMetadata( + FileSystemAbstraction fileSystemAbstraction, + DatabaseLayout databaseLayout, + Config config, + PageCache pageCache, + CursorContextFactory cursorContextFactory, + PageCacheTracer pageCacheTracer, + StoreId storeId, + UUID externalStoreId + ) { + throw new UnsupportedOperationException(); + } + + @Override + public IncrementalBatchImporter incrementalBatchImporter( + DatabaseLayout databaseLayout, + FileSystemAbstraction fileSystem, + PageCacheTracer pageCacheTracer, + Configuration config, + LogService logService, + PrintStream progressOutput, + boolean verboseProgressOutput, + AdditionalInitialIds additionalInitialIds, + ThrowingSupplier logTailMetadataSupplier, + Config dbConfig, + Monitor monitor, + JobScheduler jobScheduler, + Collector badCollector, + LogFilesInitializer logFilesInitializer, + IndexImporterFactory indexImporterFactory, + MemoryTracker memoryTracker, + CursorContextFactory contextFactory, + IndexProvidersAccess indexProvidersAccess + ) { + throw new UnsupportedOperationException(); + } + + @Override + public LockManager createLockManager(Config config, SystemNanoClock clock) { + return LockManager.NO_LOCKS_LOCK_MANAGER; + } + + @Override + public List listStorageFiles( + FileSystemAbstraction fileSystem, DatabaseLayout databaseLayout + ) { + return Collections.emptyList(); + } + + @Override + public StorageFilesState checkStoreFileState( + FileSystemAbstraction fs, DatabaseLayout databaseLayout, PageCache pageCache + ) { + return StorageFilesState.recoveredState(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java new file mode 100644 index 00000000000..e52651080d1 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageEngineImpl.java @@ -0,0 +1,341 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.configuration.Config; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.exceptions.KernelException; +import org.neo4j.gds.compat.TokenManager; +import org.neo4j.gds.config.GraphProjectConfig; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.gds.core.loading.GraphStoreCatalog; +import org.neo4j.gds.storageengine.InMemoryDatabaseCreationCatalog; +import org.neo4j.gds.storageengine.InMemoryTransactionStateVisitor; +import org.neo4j.internal.diagnostics.DiagnosticsLogger; +import org.neo4j.internal.recordstorage.InMemoryStorageReader59; +import org.neo4j.internal.schema.StorageEngineIndexingBehaviour; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.io.pagecache.tracing.DatabaseFlushEvent; +import org.neo4j.kernel.KernelVersion; +import org.neo4j.kernel.impl.store.stats.StoreEntityCounters; +import org.neo4j.kernel.lifecycle.Lifecycle; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.lock.LockGroup; +import org.neo4j.lock.LockService; +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.logging.InternalLog; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.CommandBatchToApply; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.CommandStream; +import org.neo4j.storageengine.api.IndexUpdateListener; +import org.neo4j.storageengine.api.MetadataProvider; +import org.neo4j.storageengine.api.StorageCommand; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEngineFactory; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StoreFileMetadata; +import org.neo4j.storageengine.api.StoreId; +import org.neo4j.storageengine.api.TransactionApplicationMode; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.storageengine.api.enrichment.Enrichment; +import org.neo4j.storageengine.api.enrichment.EnrichmentCommand; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; +import org.neo4j.storageengine.api.txstate.TxStateVisitor; +import org.neo4j.storageengine.api.txstate.validation.TransactionValidatorFactory; +import org.neo4j.time.SystemNanoClock; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.neo4j.gds.utils.StringFormatting.formatWithLocale; + +public final class InMemoryStorageEngineImpl implements StorageEngine { + + public static final byte ID = 42; + private final MetadataProvider metadataProvider; + private final CypherGraphStore graphStore; + private final DatabaseLayout databaseLayout; + private final InMemoryTransactionStateVisitor txStateVisitor; + + private final CommandCreationContext commandCreationContext; + + private final TokenManager tokenManager; + private final InMemoryCountsStoreImpl countsStore; + + private static final StorageEngineIndexingBehaviour INDEXING_BEHAVIOUR = new StorageEngineIndexingBehaviour() { + @Override + public boolean useNodeIdsInRelationshipTokenIndex() { + return false; + } + + @Override + public boolean requireCoordinationLocks() { + return false; + } + + @Override + public int nodesPerPage() { + return 0; + } + + @Override + public int relationshipsPerPage() { + return 0; + } + }; + + InMemoryStorageEngineImpl( + DatabaseLayout databaseLayout, + TokenHolders tokenHolders + ) { + this.databaseLayout = databaseLayout; + this.graphStore = getGraphStoreFromCatalog(databaseLayout.getDatabaseName()); + this.txStateVisitor = new InMemoryTransactionStateVisitor(graphStore, tokenHolders); + this.commandCreationContext = new InMemoryCommandCreationContextImpl(); + this.tokenManager = new TokenManager( + tokenHolders, + InMemoryStorageEngineImpl.this.txStateVisitor, + InMemoryStorageEngineImpl.this.graphStore, + commandCreationContext + ); + InMemoryStorageEngineImpl.this.graphStore.initialize(tokenHolders); + this.countsStore = new InMemoryCountsStoreImpl(graphStore, tokenHolders); + this.metadataProvider = new InMemoryMetaDataProviderImpl(); + } + + private static CypherGraphStore getGraphStoreFromCatalog(String databaseName) { + var graphName = InMemoryDatabaseCreationCatalog.getRegisteredDbCreationGraphName(databaseName); + return (CypherGraphStore) GraphStoreCatalog.getAllGraphStores() + .filter(graphStoreWithUserNameAndConfig -> graphStoreWithUserNameAndConfig + .config() + .graphName() + .equals(graphName)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(formatWithLocale( + "No graph with name `%s` was found in GraphStoreCatalog. Available graph names are %s", + graphName, + GraphStoreCatalog.getAllGraphStores() + .map(GraphStoreCatalog.GraphStoreWithUserNameAndConfig::config) + .map(GraphProjectConfig::graphName) + .collect(Collectors.toList()) + ))) + .graphStore(); + } + + @Override + public StoreEntityCounters storeEntityCounters() { + return new StoreEntityCounters() { + @Override + public long nodes() { + return graphStore.nodeCount(); + } + + @Override + public long relationships() { + return graphStore.relationshipCount(); + } + + @Override + public long properties() { + return graphStore.nodePropertyKeys().size() + graphStore.relationshipPropertyKeys().size(); + } + + @Override + public long relationshipTypes() { + return graphStore.relationshipTypes().size(); + } + + @Override + public long allNodesCountStore(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public long allRelationshipsCountStore(CursorContext cursorContext) { + return graphStore.relationshipCount(); + } + }; + } + + @Override + public void preAllocateStoreFilesForCommands( + CommandBatchToApply commandBatchToApply, + TransactionApplicationMode transactionApplicationMode + ) { + } + + @Override + public StoreCursors createStorageCursors(CursorContext initialContext) { + return StoreCursors.NULL; + } + + @Override + public StorageLocks createStorageLocks(ResourceLocker locker) { + return new InMemoryStorageLocksImpl(locker); + } + + @Override + public List createCommands( + ReadableTransactionState state, + StorageReader storageReader, + CommandCreationContext creationContext, + LockTracer lockTracer, + TxStateVisitor.Decorator additionalTxStateVisitor, + CursorContext cursorContext, + StoreCursors storeCursors, + MemoryTracker memoryTracker + ) throws KernelException { + state.accept(txStateVisitor); + return List.of(); + } + + @Override + public void dumpDiagnostics(InternalLog internalLog, DiagnosticsLogger diagnosticsLogger) { + } + + @Override + public List createUpgradeCommands( + KernelVersion versionToUpgradeFrom, + KernelVersion versionToUpgradeTo + ) { + return List.of(); + } + + @Override + public EnrichmentCommand createEnrichmentCommand(KernelVersion kernelVersion, Enrichment enrichment) { + throw new UnsupportedOperationException(); + } + + @Override + public StoreId retrieveStoreId() { + return metadataProvider.getStoreId(); + } + + @Override + public StorageEngineIndexingBehaviour indexingBehaviour() { + return INDEXING_BEHAVIOUR; + } + + @Override + public StorageReader newReader() { + return new InMemoryStorageReader59(graphStore, tokenManager.tokenHolders(), countsStore); + } + + @Override + public void addIndexUpdateListener(IndexUpdateListener listener) { + + } + + @Override + public void apply(CommandBatchToApply batch, TransactionApplicationMode mode) { + } + + @Override + public void init() { + } + + @Override + public void start() { + + } + + @Override + public void stop() { + shutdown(); + } + + @Override + public void shutdown() { + InMemoryDatabaseCreationCatalog.removeDatabaseEntry(databaseLayout.getDatabaseName()); + } + + @Override + public void listStorageFiles( + Collection atomic, Collection replayable + ) { + + } + + @Override + public Lifecycle schemaAndTokensLifecycle() { + return new LifecycleAdapter() { + @Override + public void init() { + + } + }; + } + + @Override + public CountsAccessor countsAccessor() { + return countsStore; + } + + @Override + public MetadataProvider metadataProvider() { + return metadataProvider; + } + + @Override + public String name() { + return "gds in-memory storage engine"; + } + + @Override + public byte id() { + return ID; + } + + @Override + public CommandCreationContext newCommandCreationContext() { + return commandCreationContext; + } + + @Override + public TransactionValidatorFactory createTransactionValidatorFactory(StorageEngineFactory storageEngineFactory, Config config, SystemNanoClock clock) { + return TransactionValidatorFactory.EMPTY_VALIDATOR_FACTORY; + } + + @Override + public void lockRecoveryCommands( + CommandStream commands, LockService lockService, LockGroup lockGroup, TransactionApplicationMode mode + ) { + + } + + @Override + public void rollback(ReadableTransactionState txState, CursorContext cursorContext) { + // rollback is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + // TODO: do we want to inspect the txState to infer if rollback was called explicitly or not? + } + + @Override + public void checkpoint(DatabaseFlushEvent flushEvent, CursorContext cursorContext) { + // checkpoint is not supported but it is also called when we fail for something else + // that we do not support, such as removing node properties + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java new file mode 100644 index 00000000000..0b49cdb34aa --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStorageLocksImpl.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.lock.LockTracer; +import org.neo4j.lock.ResourceLocker; +import org.neo4j.storageengine.api.StorageLocks; +import org.neo4j.storageengine.api.txstate.ReadableTransactionState; + +public class InMemoryStorageLocksImpl implements StorageLocks { + + InMemoryStorageLocksImpl(ResourceLocker locker) {} + + @Override + public void acquireExclusiveNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveNodeLock(long... ids) {} + + @Override + public void acquireSharedNodeLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedNodeLock(long... ids) {} + + @Override + public void acquireExclusiveRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseExclusiveRelationshipLock(long... ids) {} + + @Override + public void acquireSharedRelationshipLock(LockTracer lockTracer, long... ids) {} + + @Override + public void releaseSharedRelationshipLock(long... ids) {} + + @Override + public void acquireRelationshipCreationLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireRelationshipDeletionLock( + LockTracer lockTracer, + long sourceNode, + long targetNode, + long relationship, + boolean relationshipAddedInTx, + boolean sourceNodeAddedInTx, + boolean targetNodeAddedInTx + ) { + } + + @Override + public void acquireNodeDeletionLock( + ReadableTransactionState readableTransactionState, + LockTracer lockTracer, + long node + ) {} + + @Override + public void acquireNodeLabelChangeLock(LockTracer lockTracer, long node, int labelId) {} +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java new file mode 100644 index 00000000000..fcd75bd3438 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryStoreVersion.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.storageengine.api.StoreVersion; +import org.neo4j.storageengine.api.format.Capability; +import org.neo4j.storageengine.api.format.CapabilityType; + +import java.util.Optional; + +public class InMemoryStoreVersion implements StoreVersion { + + public static final String STORE_VERSION = "gds-experimental"; + + @Override + public String getStoreVersionUserString() { + return "Unknown"; + } + + @Override + public Optional successorStoreVersion() { + return Optional.empty(); + } + + @Override + public String formatName() { + return getClass().getSimpleName(); + } + + @Override + public boolean onlyForMigration() { + return false; + } + + @Override + public boolean hasCapability(Capability capability) { + return false; + } + + @Override + public boolean hasCompatibleCapabilities( + StoreVersion otherVersion, CapabilityType type + ) { + return false; + } + + @Override + public String introductionNeo4jVersion() { + return "foo"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java new file mode 100644 index 00000000000..2e220e4d3bc --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryTransactionIdStoreImpl.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.internal.recordstorage.AbstractTransactionIdStore; +import org.neo4j.io.pagecache.context.TransactionIdSnapshot; +import org.neo4j.kernel.impl.transaction.log.LogPosition; +import org.neo4j.storageengine.api.ClosedTransactionMetadata; +import org.neo4j.storageengine.api.TransactionId; +import org.neo4j.storageengine.api.TransactionIdStore; + +public class InMemoryTransactionIdStoreImpl extends AbstractTransactionIdStore { + + @Override + protected void initLastCommittedAndClosedTransactionId( + long previouslyCommittedTxId, + int checksum, + long previouslyCommittedTxCommitTimestamp, + long previouslyCommittedTxLogByteOffset, + long previouslyCommittedTxLogVersion + ) { + this.setLastCommittedAndClosedTransactionId( + previouslyCommittedTxId, + checksum, + previouslyCommittedTxCommitTimestamp, + TransactionIdStore.UNKNOWN_CONSENSUS_INDEX, + previouslyCommittedTxLogByteOffset, + previouslyCommittedTxLogVersion + ); + } + + @Override + public ClosedTransactionMetadata getLastClosedTransaction() { + long[] metaData = this.closedTransactionId.get(); + return new ClosedTransactionMetadata( + metaData[0], + new LogPosition(metaData[1], metaData[2]), + (int) metaData[3], + metaData[4], + metaData[5] + ); + } + + @Override + public TransactionIdSnapshot getClosedTransactionSnapshot() { + return new TransactionIdSnapshot(this.getLastClosedTransactionId()); + } + + @Override + protected TransactionId transactionId(long transactionId, int checksum, long commitTimestamp) { + return new TransactionId(transactionId, checksum, commitTimestamp, TransactionIdStore.UNKNOWN_CONSENSUS_INDEX); + } + + @Override + public void transactionCommitted(long transactionId, int checksum, long commitTimestamp, long consensusIndex) { + + } + + @Override + public void setLastCommittedAndClosedTransactionId( + long transactionId, + int checksum, + long commitTimestamp, + long consensusIndex, + long byteOffset, + long logVersion + ) { + + } + + @Override + public void transactionClosed( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.offer( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } + + @Override + public void resetLastClosedTransaction( + long transactionId, + long logVersion, + long byteOffset, + int checksum, + long commitTimestamp, + long consensusIndex + ) { + this.closedTransactionId.set( + transactionId, + new long[]{logVersion, byteOffset, checksum, commitTimestamp, consensusIndex} + ); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java new file mode 100644 index 00000000000..fb196c18fbe --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/InMemoryVersionCheck.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.kernel.impl.store.format.FormatFamily; +import org.neo4j.storageengine.api.StoreVersionCheck; +import org.neo4j.storageengine.api.StoreVersionIdentifier; + +import static org.neo4j.gds.compat._59.InMemoryStoreVersion.STORE_VERSION; + +public class InMemoryVersionCheck implements StoreVersionCheck { + + private static final StoreVersionIdentifier STORE_IDENTIFIER = new StoreVersionIdentifier( + STORE_VERSION, + FormatFamily.STANDARD.name(), + 0, + 0 + ); + + @Override + public boolean isCurrentStoreVersionFullySupported(CursorContext cursorContext) { + return true; + } + + @Override + public MigrationCheckResult getAndCheckMigrationTargetVersion(String formatFamily, CursorContext cursorContext) { + return new StoreVersionCheck.MigrationCheckResult(MigrationOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public UpgradeCheckResult getAndCheckUpgradeTargetVersion(CursorContext cursorContext) { + return new StoreVersionCheck.UpgradeCheckResult(UpgradeOutcome.NO_OP, STORE_IDENTIFIER, null, null); + } + + @Override + public String getIntroductionVersionFromVersion(StoreVersionIdentifier storeVersionIdentifier) { + return STORE_VERSION; + } + + public StoreVersionIdentifier findLatestVersion(String s) { + return STORE_IDENTIFIER; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java new file mode 100644 index 00000000000..81132c0c9de --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyFactoryImpl.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.annotations.service.ServiceProvider; +import org.neo4j.gds.compat.Neo4jVersion; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.compat.StorageEngineProxyFactory; + +@ServiceProvider +public class StorageEngineProxyFactoryImpl implements StorageEngineProxyFactory { + + @Override + public boolean canLoad(Neo4jVersion version) { + return version == Neo4jVersion.V_5_9; + } + + @Override + public StorageEngineProxyApi load() { + return new StorageEngineProxyImpl(); + } + + @Override + public String description() { + return "Storage Engine 5.9"; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java new file mode 100644 index 00000000000..8cda8e608ed --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/gds/compat/_59/StorageEngineProxyImpl.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.compat._59; + +import org.neo4j.common.Edition; +import org.neo4j.configuration.Config; +import org.neo4j.configuration.GraphDatabaseInternalSettings; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.dbms.api.DatabaseManagementService; +import org.neo4j.gds.compat.AbstractInMemoryNodeCursor; +import org.neo4j.gds.compat.AbstractInMemoryNodePropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipPropertyCursor; +import org.neo4j.gds.compat.AbstractInMemoryRelationshipTraversalCursor; +import org.neo4j.gds.compat.GdsDatabaseManagementServiceBuilder; +import org.neo4j.gds.compat.GraphDatabaseApiProxy; +import org.neo4j.gds.compat.StorageEngineProxyApi; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.graphdb.Direction; +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.internal.recordstorage.AbstractInMemoryRelationshipScanCursor; +import org.neo4j.internal.recordstorage.InMemoryStorageReader59; +import org.neo4j.io.layout.DatabaseLayout; +import org.neo4j.storageengine.api.CommandCreationContext; +import org.neo4j.storageengine.api.PropertySelection; +import org.neo4j.storageengine.api.RelationshipSelection; +import org.neo4j.storageengine.api.StorageEngine; +import org.neo4j.storageengine.api.StorageEntityCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.token.TokenHolders; + +import static org.neo4j.configuration.GraphDatabaseSettings.db_format; + +public class StorageEngineProxyImpl implements StorageEngineProxyApi { + + @Override + public CommandCreationContext inMemoryCommandCreationContext() { + return new InMemoryCommandCreationContextImpl(); + } + + @Override + public void initRelationshipTraversalCursorForRelType( + StorageRelationshipTraversalCursor cursor, + long sourceNodeId, + int relTypeToken + ) { + var relationshipSelection = RelationshipSelection.selection( + relTypeToken, + Direction.OUTGOING + ); + cursor.init(sourceNodeId, -1, relationshipSelection); + } + + @Override + public StorageReader inMemoryStorageReader( + CypherGraphStore graphStore, TokenHolders tokenHolders, CountsAccessor counts + ) { + return new InMemoryStorageReader59(graphStore, tokenHolders, counts); + } + + @Override + public StorageEngine createInMemoryStorageEngine(DatabaseLayout databaseLayout, TokenHolders tokenHolders) { + return new InMemoryStorageEngineImpl(databaseLayout, tokenHolders); + } + + @Override + public void createInMemoryDatabase( + DatabaseManagementService dbms, + String dbName, + Config config + ) { + config.set(db_format, InMemoryStorageEngineFactory.IN_MEMORY_STORAGE_ENGINE_NAME); + dbms.createDatabase(dbName, config); + } + + @Override + public GraphDatabaseService startAndGetInMemoryDatabase(DatabaseManagementService dbms, String dbName) { + dbms.startDatabase(dbName); + return dbms.database(dbName); + } + + @Override + public GdsDatabaseManagementServiceBuilder setSkipDefaultIndexesOnCreationSetting(GdsDatabaseManagementServiceBuilder dbmsBuilder) { + return dbmsBuilder.setConfig(GraphDatabaseInternalSettings.skip_default_indexes_on_creation, true); + } + + @Override + public AbstractInMemoryNodeCursor inMemoryNodeCursor(CypherGraphStore graphStore, TokenHolders tokenHolders) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryNodePropertyCursor inMemoryNodePropertyCursor( + CypherGraphStore graphStore, + TokenHolders tokenHolders + ) { + return new InMemoryNodePropertyCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipTraversalCursor inMemoryRelationshipTraversalCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipScanCursor inMemoryRelationshipScanCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public AbstractInMemoryRelationshipPropertyCursor inMemoryRelationshipPropertyCursor( + CypherGraphStore graphStore, TokenHolders tokenHolders + ) { + return new InMemoryRelationshipPropertyCursor(graphStore, tokenHolders); + } + + @Override + public void properties( + StorageEntityCursor storageCursor, StoragePropertyCursor propertyCursor, int[] propertySelection + ) { + PropertySelection selection; + if (propertySelection.length == 0) { + selection = PropertySelection.ALL_PROPERTIES; + } else { + selection = PropertySelection.selection(propertySelection); + } + storageCursor.properties(propertyCursor, selection); + } + + @Override + public Edition dbmsEdition(GraphDatabaseService databaseService) { + return GraphDatabaseApiProxy.dbmsInfo(databaseService).edition; + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java new file mode 100644 index 00000000000..793eae1cabd --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryLogVersionRepository59.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.storageengine.api.LogVersionRepository; + +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryLogVersionRepository59 implements LogVersionRepository { + + private final AtomicLong logVersion; + private final AtomicLong checkpointLogVersion; + + public InMemoryLogVersionRepository59() { + this(0, 0); + } + + private InMemoryLogVersionRepository59(long initialLogVersion, long initialCheckpointLogVersion) { + this.logVersion = new AtomicLong(); + this.checkpointLogVersion = new AtomicLong(); + this.logVersion.set(initialLogVersion); + this.checkpointLogVersion.set(initialCheckpointLogVersion); + } + + @Override + public void setCurrentLogVersion(long version) { + this.logVersion.set(version); + } + + @Override + public long incrementAndGetVersion() { + return this.logVersion.incrementAndGet(); + } + + @Override + public void setCheckpointLogVersion(long version) { + this.checkpointLogVersion.set(version); + } + + @Override + public long incrementAndGetCheckpointLogVersion() { + return this.checkpointLogVersion.incrementAndGet(); + } + + @Override + public long getCurrentLogVersion() { + return this.logVersion.get(); + } + + @Override + public long getCheckpointLogVersion() { + return this.checkpointLogVersion.get(); + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java new file mode 100644 index 00000000000..4aac7460e26 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageCommandReaderFactory59.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.neo4j.kernel.KernelVersion; +import org.neo4j.storageengine.api.CommandReader; +import org.neo4j.storageengine.api.CommandReaderFactory; + +public class InMemoryStorageCommandReaderFactory59 implements CommandReaderFactory { + + public static final CommandReaderFactory INSTANCE = new InMemoryStorageCommandReaderFactory59(); + + @Override + public CommandReader get(KernelVersion kernelVersion) { + switch (kernelVersion) { + case V4_2: + return LogCommandSerializationV4_2.INSTANCE; + case V4_3_D4: + return LogCommandSerializationV4_3_D3.INSTANCE; + case V5_0: + return LogCommandSerializationV5_0.INSTANCE; + default: + throw new IllegalArgumentException("Unsupported kernel version " + kernelVersion); + } + } +} diff --git a/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java new file mode 100644 index 00000000000..b7e8ea9aae7 --- /dev/null +++ b/compatibility/5.9/storage-engine-adapter/src/main/java17/org/neo4j/internal/recordstorage/InMemoryStorageReader59.java @@ -0,0 +1,332 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.internal.recordstorage; + +import org.eclipse.collections.api.set.primitive.IntSet; +import org.eclipse.collections.impl.set.immutable.primitive.ImmutableIntSetFactoryImpl; +import org.neo4j.common.EntityType; +import org.neo4j.common.TokenNameLookup; +import org.neo4j.counts.CountsAccessor; +import org.neo4j.gds.compat._59.InMemoryNodeCursor; +import org.neo4j.gds.compat._59.InMemoryPropertyCursor; +import org.neo4j.gds.compat._59.InMemoryRelationshipScanCursor; +import org.neo4j.gds.compat._59.InMemoryRelationshipTraversalCursor; +import org.neo4j.gds.core.cypher.CypherGraphStore; +import org.neo4j.internal.schema.ConstraintDescriptor; +import org.neo4j.internal.schema.IndexDescriptor; +import org.neo4j.internal.schema.IndexType; +import org.neo4j.internal.schema.SchemaDescriptor; +import org.neo4j.internal.schema.constraints.IndexBackedConstraintDescriptor; +import org.neo4j.io.pagecache.context.CursorContext; +import org.neo4j.memory.MemoryTracker; +import org.neo4j.storageengine.api.AllNodeScan; +import org.neo4j.storageengine.api.AllRelationshipsScan; +import org.neo4j.storageengine.api.StorageNodeCursor; +import org.neo4j.storageengine.api.StoragePropertyCursor; +import org.neo4j.storageengine.api.StorageReader; +import org.neo4j.storageengine.api.StorageRelationshipScanCursor; +import org.neo4j.storageengine.api.StorageRelationshipTraversalCursor; +import org.neo4j.storageengine.api.StorageSchemaReader; +import org.neo4j.storageengine.api.cursor.StoreCursors; +import org.neo4j.token.TokenHolders; + +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +public class InMemoryStorageReader59 implements StorageReader { + + protected final CypherGraphStore graphStore; + protected final TokenHolders tokenHolders; + protected final CountsAccessor counts; + private final Map, Object> dependantState; + private boolean closed; + + public InMemoryStorageReader59( + CypherGraphStore graphStore, + TokenHolders tokenHolders, + CountsAccessor counts + ) { + this.graphStore = graphStore; + + this.tokenHolders = tokenHolders; + this.counts = counts; + this.dependantState = new ConcurrentHashMap<>(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] changedLabels, + long[] unchangedLabels, + int[] propertyKeyIds, + boolean propertyKeyListIsComplete, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public long relationshipsGetCount(CursorContext cursorTracer) { + return graphStore.relationshipCount(); + } + + @Override + public boolean nodeExists(long id, StoreCursors storeCursors) { + var originalId = graphStore.nodes().toOriginalNodeId(id); + return graphStore.nodes().contains(originalId); + } + + @Override + public boolean relationshipExists(long id, StoreCursors storeCursors) { + return true; + } + + @Override + public StorageNodeCursor allocateNodeCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryNodeCursor(graphStore, tokenHolders); + } + + @Override + public StoragePropertyCursor allocatePropertyCursor( + CursorContext cursorContext, StoreCursors storeCursors, MemoryTracker memoryTracker + ) { + return new InMemoryPropertyCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipTraversalCursor allocateRelationshipTraversalCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipTraversalCursor(graphStore, tokenHolders); + } + + @Override + public StorageRelationshipScanCursor allocateRelationshipScanCursor( + CursorContext cursorContext, StoreCursors storeCursors + ) { + return new InMemoryRelationshipScanCursor(graphStore, tokenHolders); + } + + @Override + public IndexDescriptor indexGetForSchemaAndType( + SchemaDescriptor descriptor, IndexType type + ) { + return null; + } + + @Override + public AllRelationshipsScan allRelationshipScan() { + return new AbstractInMemoryAllRelationshipScan() { + @Override + boolean scanRange(AbstractInMemoryRelationshipScanCursor cursor, long start, long stopInclusive) { + return cursor.scanRange(start, stopInclusive); + } + + @Override + public boolean scanBatch(long sizeHint, AbstractInMemoryRelationshipScanCursor cursor) { + return super.scanBatch(sizeHint, cursor); + } + }; + } + + @Override + public Iterator indexGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator indexesGetForRelationshipType(int relationshipType) { + return Collections.emptyIterator(); + } + + @Override + public IndexDescriptor indexGetForName(String name) { + return null; + } + + @Override + public ConstraintDescriptor constraintGetForName(String name) { + return null; + } + + @Override + public boolean indexExists(IndexDescriptor index) { + return false; + } + + @Override + public Iterator indexesGetAll() { + return Collections.emptyIterator(); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int propertyKeyId, EntityType entityType + ) { + return valueIndexesGetRelated(tokens, new int[]{propertyKeyId}, entityType); + } + + @Override + public Collection valueIndexesGetRelated( + long[] tokens, int[] propertyKeyIds, EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] labels, + int propertyKeyId, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public Collection uniquenessConstraintsGetRelated( + long[] tokens, + int[] propertyKeyIds, + EntityType entityType + ) { + return Collections.emptyList(); + } + + @Override + public boolean hasRelatedSchema(long[] labels, int propertyKey, EntityType entityType) { + return false; + } + + @Override + public boolean hasRelatedSchema(int label, EntityType entityType) { + return false; + } + + @Override + public Iterator constraintsGetForSchema(SchemaDescriptor descriptor) { + return Collections.emptyIterator(); + } + + @Override + public boolean constraintExists(ConstraintDescriptor descriptor) { + return false; + } + + @Override + public Iterator constraintsGetForLabel(int labelId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetForRelationshipType(int typeId) { + return Collections.emptyIterator(); + } + + @Override + public Iterator constraintsGetAll() { + return Collections.emptyIterator(); + } + + @Override + public IntSet constraintsGetPropertyTokensForLogicalKey(int token, EntityType entityType) { + return ImmutableIntSetFactoryImpl.INSTANCE.empty(); + } + + @Override + public Long indexGetOwningUniquenessConstraintId(IndexDescriptor index) { + return null; + } + + @Override + public long countsForNode(int labelId, CursorContext cursorContext) { + return counts.nodeCount(labelId, cursorContext); + } + + @Override + public long countsForRelationship(int startLabelId, int typeId, int endLabelId, CursorContext cursorContext) { + return counts.relationshipCount(startLabelId, typeId, endLabelId, cursorContext); + } + + @Override + public long nodesGetCount(CursorContext cursorContext) { + return graphStore.nodeCount(); + } + + @Override + public int labelCount() { + return graphStore.nodes().availableNodeLabels().size(); + } + + @Override + public int propertyKeyCount() { + int nodePropertyCount = graphStore + .schema() + .nodeSchema() + .allProperties() + .size(); + int relPropertyCount = graphStore + .schema() + .relationshipSchema() + .allProperties() + .size(); + return nodePropertyCount + relPropertyCount; + } + + @Override + public int relationshipTypeCount() { + return graphStore.schema().relationshipSchema().availableTypes().size(); + } + + @Override + public T getOrCreateSchemaDependantState(Class type, Function factory) { + return type.cast(dependantState.computeIfAbsent(type, key -> factory.apply(this))); + } + + @Override + public AllNodeScan allNodeScan() { + return new InMemoryNodeScan(); + } + + @Override + public void close() { + assert !closed; + closed = true; + } + + @Override + public StorageSchemaReader schemaSnapshot() { + return this; + } + + @Override + public TokenNameLookup tokenNameLookup() { + return tokenHolders; + } + +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 7e41ca69b83..d2d98bb716e 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -9,6 +9,7 @@ ext { '5.6': properties.getOrDefault('neo4jVersion56', '5.6.0'), '5.7': properties.getOrDefault('neo4jVersion57', '5.7.0'), '5.8': properties.getOrDefault('neo4jVersion58', '5.8.0'), + '5.9': properties.getOrDefault('neo4jVersion59', '5.9.0'), ] neo4jDefault = neos.'4.4' diff --git a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java index 6f31ca9ab5c..08b57c31b6d 100644 --- a/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java +++ b/neo4j-adapter/src/main/java/org/neo4j/gds/compat/Neo4jVersion.java @@ -36,6 +36,7 @@ public enum Neo4jVersion { V_5_6, V_5_7, V_5_8, + V_5_9, V_RC; @Override @@ -59,6 +60,8 @@ public String toString() { return "5.7"; case V_5_8: return "5.8"; + case V_5_9: + return "5.9"; case V_RC: return "rc"; default: @@ -68,7 +71,7 @@ public String toString() { public MajorMinorVersion semanticVersion() { if (this == V_RC) { - return ImmutableMajorMinorVersion.of(5, 9); + return ImmutableMajorMinorVersion.of(5, 10); } String version = toString(); var subVersions = version.split("\\."); @@ -147,6 +150,8 @@ static Neo4jVersion parse(String version) { } else if (minorVersion == 8) { return Neo4jVersion.V_5_8; } else if (minorVersion == 9) { + return Neo4jVersion.V_5_9; + } else if (minorVersion == 10) { return Neo4jVersion.V_RC; } } diff --git a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java index 999179b6812..247015c722d 100644 --- a/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java +++ b/neo4j-adapter/src/test/java/org/neo4j/gds/compat/Neo4jVersionTest.java @@ -49,7 +49,8 @@ class Neo4jVersionTest { "5.6.0, V_5_6", "5.7.0, V_5_7", "5.8.0, V_5_8", - "5.9.0, V_RC", + "5.9.0, V_5_9", + "5.10.0, V_RC", }) void testParse(String input, Neo4jVersion expected) { assertEquals(expected.name(), Neo4jVersion.parse(input).name()); @@ -90,6 +91,7 @@ void shouldNotRespectVersionOverride() { "5.6.0, 5, 6", "5.7.0, 5, 7", "5.8.0, 5, 8", + "5.9.0, 5, 9", }) void semanticVersion(String input, int expectedMajor, int expectedMinor) { Neo4jVersion version = Neo4jVersion.parse(input); diff --git a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java index 0c5e0e4d2be..a49229caf17 100644 --- a/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java +++ b/proc/sysinfo/src/test/java/org/neo4j/gds/SysInfoProcTest.java @@ -91,6 +91,11 @@ class SysInfoProcTest extends BaseProcTest { "Neo4j Settings 5.8", "Neo4j Settings 5.8 (placeholder)", + "Neo4j 5.9", + "Neo4j 5.9 (placeholder)", + "Neo4j Settings 5.9", + "Neo4j Settings 5.9 (placeholder)", + "Neo4j RC", "Neo4j RC (placeholder)", "Neo4j Settings RC", @@ -210,6 +215,14 @@ void testSysInfoProc() throws IOException { "Neo4j 5.8" ); break; + case V_5_9: + expectedCompatibilities = Set.of( + "Neo4j Settings 5.9 (placeholder)", + "Neo4j Settings 5.9", + "Neo4j 5.9 (placeholder)", + "Neo4j 5.9" + ); + break; case V_RC: expectedCompatibilities = Set.of( "Neo4j Settings RC (placeholder)", From 745801384a0d323176d34381a4e1cf787267390c Mon Sep 17 00:00:00 2001 From: Mats Rydberg Date: Mon, 5 Jun 2023 17:33:38 +0200 Subject: [PATCH 391/400] Bump Aura version number --- gradle/version.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/version.gradle b/gradle/version.gradle index 929fbf0acdd..c9ebf7ac4ef 100644 --- a/gradle/version.gradle +++ b/gradle/version.gradle @@ -1,6 +1,6 @@ ext { gdsBaseVersion = '2.3.9' - gdsAuraVersion = '22' + gdsAuraVersion = '23' gdsVersion = gdsBaseVersion + (rootProject.hasProperty('aurads') ? "+${gdsAuraVersion}" : "") } From 81d0f47c77dad62d3cbfcc79f9e59cfd8fe42527 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Tue, 6 Jun 2023 06:44:35 +0100 Subject: [PATCH 392/400] Point to analytical cluster setup docs --- doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index f7b5b47933b..a66a34cba84 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -29,7 +29,7 @@ GDS has compute-intensive OLAP workloads that may disrupt the cluster operations [NOTE] ====== -Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/[official Neo4j documentation] for details on how to setup Neo4j cluster. +Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/setup/analytics-cluster/[official Neo4j documentation] for details on how to setup Neo4j analytical cluster. Note that the link points to the latest Neo4j version documentation and the configuration settings may differ from earlier versions. ====== From 6f580ca8cce689ef7829731ab7a1d7dfaf6f7850 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 8 Jun 2023 07:15:36 +0100 Subject: [PATCH 393/400] Improve wording Co-authored-by: Jessica Wright <49636617+AlexicaWright@users.noreply.github.com> --- doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc index a66a34cba84..3c4fd176d11 100644 --- a/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc +++ b/doc/modules/ROOT/pages/production-deployment/neo4j-cluster.adoc @@ -29,7 +29,7 @@ GDS has compute-intensive OLAP workloads that may disrupt the cluster operations [NOTE] ====== -Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/setup/analytics-cluster/[official Neo4j documentation] for details on how to setup Neo4j analytical cluster. +Please refer to the https://neo4j.com/docs/operations-manual/current/clustering/setup/analytics-cluster/[official Neo4j documentation] for details on how to set up a Neo4j analytics cluster. Note that the link points to the latest Neo4j version documentation and the configuration settings may differ from earlier versions. ====== From d4bdc91286f7618a682bb08f71ca339eda8ac497 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 7 Jun 2023 13:12:17 +0200 Subject: [PATCH 394/400] Add missing syntaxt testing --- .../org/neo4j/gds/doc/syntax/SyntaxMode.java | 5 ++- .../doc/syntax/GraphGenerationSyntaxTest.java | 37 +++++++++++++++++++ .../projections/graph-generation.adoc | 14 ++++--- 3 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java index a200a2ed2dd..9eb1c43135a 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java @@ -66,7 +66,10 @@ public enum SyntaxMode { SYSTEM_MONITOR("system-monitor-syntax", false), SYS_INFO("debug-sysinfo-syntax", false), WRITE_NODE_LABEL("include-with-write-node-label", false), - MUTATE_NODE_LABEL("include-with-mutate-node-label", false),; + MUTATE_NODE_LABEL("include-with-mutate-node-label", false), + GRAPH_GENERATE("include-with-graph-generate"), + ; + private final String mode; public final boolean hasParameters; diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java new file mode 100644 index 00000000000..20ca0ed6b04 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/syntax/GraphGenerationSyntaxTest.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc.syntax; + +import java.util.List; + +import static org.neo4j.gds.doc.syntax.SyntaxMode.GRAPH_GENERATE; + +class GraphGenerationSyntaxTest extends SyntaxTestBase { + + @Override + protected Iterable syntaxModes() { + return List.of(SyntaxModeMeta.of(GRAPH_GENERATE)); + } + + @Override + protected String adocFile() { + return "pages/management-ops/projections/graph-generation.adoc"; + } +} diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc index c484061a1c8..49a396ceff0 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc @@ -26,13 +26,16 @@ The graph generation is parameterized by three dimensions: [[graph-generation-syntax]] == Syntax - -.The following describes the API for running the algorithm +[.include-with-graph-generate] +==== +.The following describes the API for running the graph generation procedure [source, cypher, role=noplay] ---- -CALL gds.beta.graph.generate(graphName: String, nodeCount: Integer, averageDegree: Integer, { - relationshipDistribution: String, - relationshipProperty: Map +CALL gds.beta.graph.generate( + graphName: String, + nodeCount: Integer, + averageDegree: Integer, + configuration: Map }) YIELD name, nodes, relationships, generateMillis, relationshipSeed, averageDegree, relationshipDistribution, relationshipProperty ---- @@ -72,6 +75,7 @@ YIELD name, nodes, relationships, generateMillis, relationshipSeed, averageDegre | relationshipDistribution | String | The probability distribution method used to connect generated nodes. | relationshipProperty | String | The configuration of the generated relationship property. |=== +==== [[graph-generation-distribution]] == Relationship Distribution From 40acb4a21a139811fb5867c811444f25a6599e47 Mon Sep 17 00:00:00 2001 From: ioannispan Date: Wed, 7 Jun 2023 14:44:04 +0200 Subject: [PATCH 395/400] Add example --- .../neo4j/gds/doc/GraphGenerationDocTest.java | 47 ++++++++ .../projections/graph-generation.adoc | 112 ++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java diff --git a/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java b/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java new file mode 100644 index 00000000000..18b3212ebc4 --- /dev/null +++ b/doc-test/src/test/java/org/neo4j/gds/doc/GraphGenerationDocTest.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [http://neo4j.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.gds.doc; + +import org.neo4j.gds.beta.generator.GraphGenerateProc; +import org.neo4j.gds.catalog.GraphStreamRelationshipPropertiesProc; +import org.neo4j.gds.catalog.GraphStreamRelationshipsProc; +import org.neo4j.gds.functions.AsNodeFunc; + +import java.util.List; + +class GraphGenerationDocTest extends SingleFileDocTestBase { + + @Override + protected List> functions() { + return List.of(AsNodeFunc.class); + } + + @Override + protected List> procedures() { + return List.of(GraphGenerateProc.class, GraphStreamRelationshipsProc.class, + GraphStreamRelationshipPropertiesProc.class + ); + } + + @Override + protected String adocFile() { + return "pages/management-ops/projections/graph-generation.adoc"; + } +} diff --git a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc index 49a396ceff0..8c1b7d2d7c3 100644 --- a/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc +++ b/doc/modules/ROOT/pages/management-ops/projections/graph-generation.adoc @@ -116,3 +116,115 @@ Currently, there are two supported methods to generate relationship properties: * `FIXED` - Assigns a fixed value to every relationship. The `value` parameter must be set. * `RANDOM` - Assigns a random value between the lower (`min`) and upper (`max`) bound. + +[[graph-generation-example]] +== Examples + +In the following we will demonstrate the usage of the random graph generation procedure. + +[[graph-generation-unweighted]] +=== Generating unweighted graphs + +[role=query-example,group=unweighted] +-- +.The following will produce a graph with unweighted relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.generate('graph',5,2, {relationshipSeed:19}) +YIELD name, nodes, relationships, relationshipDistribution +---- + +.Results +[opts="header"] +|=== +| name | nodes | relationships | relationshipDistribution +| "graph"| 5 | 10 | "UNIFORM" +|=== +-- + +A new in-memory graph called `graph` with `5` nodes and `10` relationships has been created and added to the graph catalog. +We can examine its topology with the `gds.beta.graph.relationships` procedure. + +[role=query-example,group=unweighted] +-- +.The following will show the produced relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.relationships.stream('graph') +YIELD sourceNodeId,targetNodeId +RETURN sourceNodeId as source, targetNodeId as target +ORDER BY source ASC,target ASC +---- + +.Results +[opts="header"] +|=== +| source |target +| 0 | 1 +| 0 | 2 +| 1 | 0 +| 1 | 4 +| 2 | 1 +| 2 | 4 +| 3 | 0 +| 3 | 1 +| 4 | 0 +| 4 | 3 +|=== +-- + +[[graph-generation-weighted]] +=== Generating weighted graphs + +To generated graphs with weighted relationships we must specify the `relationshipProperty` parameter as discussed xref:graph-generation-relationship-property[above]. + +[role=query-example,group=weighted] +-- +.The following will produce a graph with weighted relationships +[source,cypher,role=noplay] +---- +CALL gds.beta.graph.generate('weightedGraph',5,2, {relationshipSeed:19, + relationshipProperty: {type: 'RANDOM', min: 5.0, max: 10.0, name: 'score'}}) +YIELD name, nodes, relationships, relationshipDistribution +---- + +.Results +[opts="header"] +|=== +|name| nodes | relationships | relationshipDistribution +| "weightedGraph"| 5 | 10 | "UNIFORM" +|=== +-- + +The produced graph, `weightedGraph`, has a property named `score` containing a random value between 5.0 and 10.0 for each relationship. +We can use `gds.graph.relationshipProperty.stream` to stream the relationships of the graph along with their score values. + +[role=query-example,group=weighted] +-- +.The following will show the produced relationships +[source,cypher,role=noplay] +---- +CALL gds.graph.relationshipProperty.stream('weightedGraph','score') +YIELD sourceNodeId, targetNodeId, propertyValue +RETURN sourceNodeId as source, targetNodeId as target, propertyValue as score +ORDER BY source ASC,target ASC, score +---- + +.Results +[opts="header"] +|=== +| source |target | score +| 0 | 1 | 6.258381821615686 +| 0 | 2 | 6.791408433596591 +| 1 | 4 | 8.747179900968224 +| 1 | 4 | 9.469695236791349 +| 2 | 0 | 7.061710127800056 +| 2 | 1 | 5.060444167785128 +| 3 | 1 | 6.308266834622538 +| 3 | 4 | 9.040323743901354 +| 4 | 1 | 7.939688205556302 +| 4 | 1 | 7.988277646384441 +|=== +-- + +Notice that despite `graph` and `weightedGraph` having the same `relationshipSeed`, their actual topology differs. From f375260af06150fa3220d6694740600f0d896cb7 Mon Sep 17 00:00:00 2001 From: Ioannis Pan <74839024+IoannisPanagiotas@users.noreply.github.com> Date: Wed, 7 Jun 2023 15:07:29 +0200 Subject: [PATCH 396/400] Update SyntaxMode.java --- .../src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java index 9eb1c43135a..8e80e69244d 100644 --- a/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java +++ b/doc-test-tools/src/main/java/org/neo4j/gds/doc/syntax/SyntaxMode.java @@ -67,10 +67,8 @@ public enum SyntaxMode { SYS_INFO("debug-sysinfo-syntax", false), WRITE_NODE_LABEL("include-with-write-node-label", false), MUTATE_NODE_LABEL("include-with-mutate-node-label", false), - GRAPH_GENERATE("include-with-graph-generate"), - ; - - + GRAPH_GENERATE("include-with-graph-generate"),; + private final String mode; public final boolean hasParameters; From bd030154b659d5f203456c48deb8bf5a3e16777f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Fri, 9 Jun 2023 11:31:22 +0200 Subject: [PATCH 397/400] Add 5.8 compat to public build Co-authored-by: Yuval Rotenberg --- build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle b/build.gradle index 52b8512d36b..165c9eec480 100644 --- a/build.gradle +++ b/build.gradle @@ -36,6 +36,7 @@ ext { project(':neo4j-kernel-adapter-5.5'), project(':neo4j-kernel-adapter-5.6'), project(':neo4j-kernel-adapter-5.7'), + project(':neo4j-kernel-adapter-5.8'), ], 'storage-engine-adapter': [ project(':storage-engine-adapter-4.4'), @@ -46,6 +47,7 @@ ext { project(':storage-engine-adapter-5.5'), project(':storage-engine-adapter-5.6'), project(':storage-engine-adapter-5.7'), + project(':storage-engine-adapter-5.8'), ] ] } From 1212ff06093aa2e03c039924c03164d7845cfaf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20J=C3=A4derberg?= Date: Mon, 22 May 2023 12:44:54 +0200 Subject: [PATCH 398/400] Public 5.8 compat --- README.adoc | 2 +- settings.gradle | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.adoc b/README.adoc index e8b9671cc02..f7b936de08e 100644 --- a/README.adoc +++ b/README.adoc @@ -84,7 +84,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.3.0 |Neo4j 5.4.0 .9+<.^|GDS 2.3.x -|Neo4j 4.4.9 - 4.4.21 +|Neo4j 4.4.9 - 4.4.20 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 diff --git a/settings.gradle b/settings.gradle index 6b6ada88d92..5f37fd599a6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -145,6 +145,9 @@ project(':neo4j-kernel-adapter-5.6').projectDir = file('compatibility/5.6/neo4j- include('neo4j-kernel-adapter-5.7') project(':neo4j-kernel-adapter-5.7').projectDir = file('compatibility/5.7/neo4j-kernel-adapter') +include('neo4j-kernel-adapter-5.8') +project(':neo4j-kernel-adapter-5.8').projectDir = file('compatibility/5.8/neo4j-kernel-adapter') + include('neo4j-kernel-adapter-api') project(':neo4j-kernel-adapter-api').projectDir = file('compatibility/api/neo4j-kernel-adapter') @@ -238,6 +241,9 @@ project(':storage-engine-adapter-5.6').projectDir = file('compatibility/5.6/stor include('storage-engine-adapter-5.7') project(':storage-engine-adapter-5.7').projectDir = file('compatibility/5.7/storage-engine-adapter') +include('storage-engine-adapter-5.8') +project(':storage-engine-adapter-5.8').projectDir = file('compatibility/5.8/storage-engine-adapter') + include('storage-engine-adapter-api') project(':storage-engine-adapter-api').projectDir = file('compatibility/api/storage-engine-adapter') From ead943ee8f051807c6f2c5d770160f1bbd5ad80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florentin=20D=C3=B6rre?= Date: Wed, 28 Jun 2023 09:58:14 +0200 Subject: [PATCH 399/400] Bump Neo4j 4.4 version used Co-authored-by: Brian Shi --- README.adoc | 2 +- gradle/dependencies.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.adoc b/README.adoc index f7b936de08e..85e0362cd8a 100644 --- a/README.adoc +++ b/README.adoc @@ -84,7 +84,7 @@ If you are using Neo4j Desktop you can simply add the Graph Data Science library |Neo4j 5.3.0 |Neo4j 5.4.0 .9+<.^|GDS 2.3.x -|Neo4j 4.4.9 - 4.4.20 +|Neo4j 4.4.9 - 4.4.22 |Neo4j 5.1.0 |Neo4j 5.2.0 |Neo4j 5.3.0 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index d2d98bb716e..34cecb93311 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -1,6 +1,6 @@ ext { neos = [ - '4.4': properties.getOrDefault('neo4jVersion44', '4.4.21'), + '4.4': properties.getOrDefault('neo4jVersion44', '4.4.22'), '5.1': properties.getOrDefault('neo4jVersion51', '5.1.0'), '5.2': properties.getOrDefault('neo4jVersion52', '5.2.0'), '5.3': properties.getOrDefault('neo4jVersion53', '5.3.0'), From 402932cdccc56105523978c449b222a486d95f77 Mon Sep 17 00:00:00 2001 From: Veselin Nikolov Date: Thu, 29 Jun 2023 05:10:22 +0100 Subject: [PATCH 400/400] Don't assert the wcc seeded result in docs --- doc/modules/ROOT/pages/algorithms/wcc.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/modules/ROOT/pages/algorithms/wcc.adoc b/doc/modules/ROOT/pages/algorithms/wcc.adoc index dc8a970fe48..0e4c3d3835f 100644 --- a/doc/modules/ROOT/pages/algorithms/wcc.adoc +++ b/doc/modules/ROOT/pages/algorithms/wcc.adoc @@ -522,7 +522,7 @@ CALL gds.graph.project( _Step 4_ -[role=query-example, group=seeding] +[role=query-example, group=seeding, no-result=true] -- .The following will run the algorithm in `stream` mode using `seedProperty`: [source, cypher, role=noplay]